diff options
Diffstat (limited to 'collaborativeagents/slurm')
122 files changed, 33745 insertions, 0 deletions
diff --git a/collaborativeagents/slurm/all_memory_15640363.err b/collaborativeagents/slurm/all_memory_15640363.err new file mode 100644 index 0000000..f84e5ce --- /dev/null +++ b/collaborativeagents/slurm/all_memory_15640363.err @@ -0,0 +1,544 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:06<00:19, 6.59s/it] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:08<00:08, 4.03s/it] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:14<00:04, 4.81s/it] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:20<00:00, 5.22s/it] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:20<00:00, 5.11s/it] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 13.46it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:03, 14.38it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 15.20it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 16.08it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 17.70it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 18.64it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.46it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.50it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.18it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.56it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:01, 20.37it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 21.23it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 22.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 22.64it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:02<00:00, 23.23it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 23.85it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 23.72it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 24.32it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 20.77it/s] +[0;36m(EngineCore_DP0 pid=2387432)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:04, 8.04it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:01, 16.80it/s]
Capturing CUDA graphs (decode, FULL): 20%|██ | 7/35 [00:00<00:01, 19.57it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:01, 20.96it/s]
Capturing CUDA graphs (decode, FULL): 37%|███▋ | 13/35 [00:00<00:01, 21.87it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:00<00:00, 22.53it/s]
Capturing CUDA graphs (decode, FULL): 54%|█████▍ | 19/35 [00:00<00:00, 22.90it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:01<00:00, 23.18it/s]
Capturing CUDA graphs (decode, FULL): 71%|███████▏ | 25/35 [00:01<00:00, 23.43it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:01<00:00, 23.71it/s]
Capturing CUDA graphs (decode, FULL): 89%|████████▊ | 31/35 [00:01<00:00, 23.98it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:01<00:00, 24.13it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 22.47it/s] +[0;36m(APIServer pid=2386991)[0;0m INFO: Started server process [2386991] +[0;36m(APIServer pid=2386991)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=2386991)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-25 16:16:31,965 - INFO - Loaded dataset: math-hard +2026-01-25 16:16:31,965 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-25 16:16:32,046 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-25 16:16:32,047 - INFO - Running method: all_memory +2026-01-25 16:16:32,049 - INFO - Running profiles 0 to 19 (20 remaining) +2026-01-25 16:16:32,049 - INFO - Using BATCH processing (async OpenAI user) for all_memory +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:01<00:03, 1.27s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:02<00:02, 1.28s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:04<00:01, 1.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:04<00:00, 1.10s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:00<00:03, 1.31it/s]
Loading checkpoint shards: 40%|████ | 2/5 [00:01<00:02, 1.11it/s]
Loading checkpoint shards: 60%|██████ | 3/5 [00:02<00:01, 1.04it/s]
Loading checkpoint shards: 80%|████████ | 4/5 [00:03<00:00, 1.09it/s]
Loading checkpoint shards: 100%|██████████| 5/5 [00:03<00:00, 1.37it/s] +2026-01-25 16:18:14,224 - INFO - Batch: 20 profiles × 5 sessions, max 8 turns/session +2026-01-25 16:18:31,022 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:34,489 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:34,992 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:35,261 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:36,629 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:36,737 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:36,771 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:37,390 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:38,674 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:39,997 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:40,081 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:40,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:40,748 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:43,384 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:43,824 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:44,395 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:52,791 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:18:59,937 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:02,809 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:05,121 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2026-01-25 16:21:53,541 - INFO - Retrying request to /chat/completions in 0.466663 seconds +2026-01-25 16:22:07,442 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:25,768 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:38,209 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:39,797 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:42,419 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:46,013 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:47,281 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:50,381 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:51,224 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:53,169 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:53,570 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:54,821 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:59,704 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:00,279 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:02,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:04,294 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:04,981 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:07,284 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:07,987 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:21,162 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:24:32,911 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:26:29,523 - INFO - Retrying request to /chat/completions in 0.428869 seconds +2026-01-25 16:27:05,613 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:10,306 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:13,485 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:15,659 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:21,331 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:27,924 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:28,477 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:30,915 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:31,229 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:33,973 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:38,890 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:44,475 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:48,948 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:29:01,076 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:30:26,648 - INFO - Retrying request to /chat/completions in 0.482478 seconds +2026-01-25 16:30:56,485 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:02,564 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:02,924 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:08,189 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:11,549 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:19,981 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:23,263 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:25,040 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:35,051 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:37,472 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:44,235 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:32:32,104 - INFO - Retrying request to /chat/completions in 0.429493 seconds +2026-01-25 16:32:58,118 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:02,720 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:22,380 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:33,336 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:41,804 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:48,349 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:34:26,732 - INFO - Retrying request to /chat/completions in 0.461696 seconds +2026-01-25 16:34:53,481 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:02,567 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:08,370 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:19,318 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:22,279 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:58,111 - INFO - Retrying request to /chat/completions in 0.469936 seconds +2026-01-25 16:36:23,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:36:57,102 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:37:09,336 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:37:22,762 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:38:03,404 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:38:30,070 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:03,944 - INFO - Retrying request to /chat/completions in 0.418331 seconds +2026-01-25 16:39:39,913 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:43,812 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:16,082 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:41:15,121 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:41:38,270 - INFO - Session round 1/5: 20 total, 48 sessions/hr +2026-01-25 16:41:38,273 - INFO - Retrying request to /chat/completions in 0.428800 seconds +2026-01-25 16:41:57,584 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:00,293 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:00,548 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:01,640 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:01,975 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:04,853 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:05,421 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:05,529 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:05,747 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:05,832 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:09,402 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 502 Bad Gateway" +2026-01-25 16:42:09,403 - INFO - Retrying request to /chat/completions in 0.906045 seconds +2026-01-25 16:42:16,881 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:18,672 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:19,019 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:19,248 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:21,523 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:27,904 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:29,484 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:34,154 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:42:44,903 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:43:08,142 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:43:16,232 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:02,980 - INFO - Retrying request to /chat/completions in 0.493077 seconds +2026-01-25 16:46:16,995 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:40,887 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:42,929 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:44,762 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:50,387 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:50,813 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:54,900 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:55,820 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:58,278 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:06,006 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:07,350 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:08,289 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:10,358 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:11,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:14,295 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:14,482 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:15,152 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:15,429 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:19,438 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:22,400 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:49,244 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:57,224 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:05,848 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:13,372 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:15,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:20,943 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:23,223 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:28,476 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:28,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:30,873 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:48:37,550 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:48,688 - INFO - Retrying request to /chat/completions in 0.451434 seconds +2026-01-25 16:51:11,000 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:11,582 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:16,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:16,986 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:19,016 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:21,726 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:27,295 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:31,772 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:33,594 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:40,471 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:45,369 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:45,391 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:49,764 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:53,052 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:57,917 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:00,098 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:03,891 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:43,597 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:53,447 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:53:08,203 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:53:34,872 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:53:35,521 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:54:31,234 - INFO - Retrying request to /chat/completions in 0.377739 seconds +2026-01-25 16:54:56,335 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:11,405 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:13,460 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:20,006 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:35,617 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:35,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:40,264 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:44,816 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:21,720 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:34,147 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:58,984 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:29,416 - INFO - Retrying request to /chat/completions in 0.401322 seconds +2026-01-25 16:58:08,227 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:09,154 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:12,160 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:34,378 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:27,407 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:44,406 - INFO - Retrying request to /chat/completions in 0.467862 seconds +2026-01-25 17:00:21,372 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:25,526 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:44,369 - INFO - Retrying request to /chat/completions in 0.403819 seconds +2026-01-25 17:01:24,665 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:01:24,998 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:01:46,508 - INFO - Retrying request to /chat/completions in 0.414245 seconds +2026-01-25 17:02:21,543 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:31,856 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:50,524 - INFO - Session round 2/5: 40 total, 52 sessions/hr +2026-01-25 17:02:50,527 - INFO - Retrying request to /chat/completions in 0.455748 seconds +2026-01-25 17:03:11,370 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:12,628 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:12,869 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:13,034 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:16,141 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:16,891 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:17,039 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:18,571 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:18,647 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:19,836 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:20,349 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:21,716 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:21,774 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:22,198 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:22,337 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:23,303 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:25,867 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:26,617 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:27,574 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:29,422 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:18,012 - INFO - Retrying request to /chat/completions in 0.419427 seconds +2026-01-25 17:06:30,999 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:31,755 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:37,449 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:45,398 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:49,040 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:54,007 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:54,719 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:55,083 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:55,552 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:55,682 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:58,638 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:01,711 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:01,733 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:03,111 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:06,179 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:08,049 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:08,212 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:08,918 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:08,967 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:13,714 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:20,777 - INFO - Retrying request to /chat/completions in 0.465437 seconds +2026-01-25 17:09:43,147 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:54,973 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:56,656 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:59,640 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:02,198 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:03,050 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:03,353 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:03,869 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:06,630 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:07,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:08,552 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:10,625 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:12,274 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:18,609 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:27,735 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:11:58,044 - INFO - Retrying request to /chat/completions in 0.477912 seconds +2026-01-25 17:12:19,322 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:26,089 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:27,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:33,606 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:47,293 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:50,254 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:51,297 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:51,623 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:55,112 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:01,506 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:21,917 - INFO - Retrying request to /chat/completions in 0.483742 seconds +2026-01-25 17:14:46,305 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:51,062 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:55,483 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:04,363 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:14,027 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:16,915 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:22,545 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:36,658 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:11,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:14,397 - INFO - Retrying request to /chat/completions in 0.408490 seconds +2026-01-25 17:17:33,426 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:35,075 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:41,338 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:45,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:48,715 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:49,478 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:34,515 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:22,988 - INFO - Retrying request to /chat/completions in 0.422999 seconds +2026-01-25 17:19:49,739 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:49,811 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:52,980 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:20:15,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:20:16,121 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:20:53,978 - INFO - Retrying request to /chat/completions in 0.445960 seconds +2026-01-25 17:21:19,056 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:19,314 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:31,233 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:34,786 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:15,002 - INFO - Session round 3/5: 60 total, 55 sessions/hr +2026-01-25 17:22:15,006 - INFO - Retrying request to /chat/completions in 0.477922 seconds +2026-01-25 17:22:31,368 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:34,117 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:39,438 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:39,608 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:40,342 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:40,600 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:40,662 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:40,734 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:42,690 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:43,818 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:44,013 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:44,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:45,704 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:45,825 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:46,099 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:49,464 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:52,153 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:53,226 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:55,753 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:57,386 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:25:39,951 - INFO - Retrying request to /chat/completions in 0.411565 seconds +2026-01-25 17:26:05,633 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:08,790 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:10,263 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:17,763 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:18,449 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:19,259 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:19,759 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:20,077 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:21,601 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:21,680 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:22,022 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:23,342 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:27,621 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:27,628 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:28,494 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:28,776 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:31,376 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:32,434 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:34,739 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:36,231 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:01,108 - INFO - Retrying request to /chat/completions in 0.493455 seconds +2026-01-25 17:29:17,219 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:23,777 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:29,251 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:30,044 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:30,065 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:30,224 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:38,805 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:43,655 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:44,696 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:46,595 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:46,976 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:48,674 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:52,806 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:59,205 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:01,074 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:09,409 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:21,204 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:01,098 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:06,647 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:33,189 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:06,436 - INFO - Retrying request to /chat/completions in 0.463545 seconds +2026-01-25 17:33:35,655 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:40,670 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:44,415 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:46,696 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:51,743 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:52,661 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:52,846 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:57,575 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:03,980 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:08,283 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:15,903 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:13,908 - INFO - Retrying request to /chat/completions in 0.488125 seconds +2026-01-25 17:35:40,480 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:42,782 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:45,403 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:03,537 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:08,114 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:08,912 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:10,012 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:50,335 - INFO - Retrying request to /chat/completions in 0.496696 seconds +2026-01-25 17:37:10,536 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:37:23,284 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:37:30,400 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:37:40,113 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:37:45,227 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:18,352 - INFO - Retrying request to /chat/completions in 0.485117 seconds +2026-01-25 17:38:45,188 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:51,513 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:56,823 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:02,014 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:36,382 - INFO - Retrying request to /chat/completions in 0.416719 seconds +2026-01-25 17:39:58,754 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:03,806 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:25,382 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:39,653 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:00,777 - INFO - Session round 4/5: 80 total, 57 sessions/hr +2026-01-25 17:41:00,780 - INFO - Retrying request to /chat/completions in 0.394649 seconds +2026-01-25 17:41:22,648 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:29,013 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:31,943 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:32,073 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:33,439 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:33,685 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:33,733 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:36,022 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:36,175 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:38,016 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:39,032 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:39,422 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:40,284 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:43,982 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:48,708 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:49,912 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:50,643 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:50,948 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:53,972 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:58,340 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:15,522 - INFO - Retrying request to /chat/completions in 0.472389 seconds +2026-01-25 17:45:31,766 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:32,873 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:36,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:39,035 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:42,637 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:48,462 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:51,449 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:59,347 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:02,665 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:03,615 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:07,426 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:08,671 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:12,643 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:12,673 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:12,810 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:19,043 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:20,087 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:22,879 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:24,929 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:25,334 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:47:14,446 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:59,175 - INFO - Retrying request to /chat/completions in 0.484441 seconds +2026-01-25 17:50:29,106 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:31,431 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:38,253 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:38,280 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:39,414 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:40,861 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:43,954 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:52,249 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:52,494 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:50:55,338 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:02,784 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:04,948 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:05,912 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:09,554 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:12,326 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:15,483 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:08,568 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:16,942 - INFO - Retrying request to /chat/completions in 0.495272 seconds +2026-01-25 17:54:38,947 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:48,681 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:48,982 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:52,227 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:52,379 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:03,900 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:06,025 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:20,206 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:22,614 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:23,397 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:34,065 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:35,741 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:56:20,035 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:19,849 - INFO - Retrying request to /chat/completions in 0.398933 seconds +2026-01-25 17:58:50,885 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:52,995 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:53,063 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:57,560 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:57,758 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:00,164 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:18,338 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:22,185 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:34,265 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:00:19,905 - INFO - Retrying request to /chat/completions in 0.413578 seconds +2026-01-25 18:01:36,871 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:14,756 - INFO - Retrying request to /chat/completions in 0.478863 seconds +2026-01-25 18:04:10,764 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:21,421 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:21,467 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:21,938 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:25,034 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:37,882 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:40,540 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:50,287 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:56,616 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:06:12,643 - INFO - Retrying request to /chat/completions in 0.419640 seconds +2026-01-25 18:06:50,328 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:16,568 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:18,978 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:30,816 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:32,784 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:08:03,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:08:11,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:09:06,020 - INFO - Retrying request to /chat/completions in 0.433842 seconds +2026-01-25 18:09:41,474 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:09:47,645 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:09:51,929 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:10:00,188 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:10:09,956 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:10:53,106 - INFO - Session round 5/5: 100 total, 52 sessions/hr +2026-01-25 18:10:53,611 - INFO - GPU memory freed after all_memory: 0.0GB allocated +2026-01-25 18:10:53,616 - INFO - Report saved to ../results/gpt_user_scale_all_memory_20260125_161623/20260125_161631/report.md diff --git a/collaborativeagents/slurm/fullscale/continue_to_30sess.sh b/collaborativeagents/slurm/fullscale/continue_to_30sess.sh new file mode 100644 index 0000000..93ddecc --- /dev/null +++ b/collaborativeagents/slurm/fullscale/continue_to_30sess.sh @@ -0,0 +1,76 @@ +#!/bin/bash +#SBATCH --job-name=continue_30sess +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=16:00:00 +#SBATCH --output=continue_30sess_%j.out +#SBATCH --error=continue_30sess_%j.err + +# Continue experiment from 15 to 30 sessions +# This will pick up from existing checkpoints and only run remaining sessions + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +# Find the existing results directory +RESULTS_DIR=$(ls -td ../results/fullscale_15sess/*/ 2>/dev/null | head -1) + +if [ -z "$RESULTS_DIR" ]; then + echo "ERROR: No existing results directory found in fullscale_15sess/" + exit 1 +fi + +echo "Continuing from: $RESULTS_DIR" + +# Continue with 30 sessions (the checkpoint system will skip already-completed sessions) +python run_experiments.py \ + --methods vanilla,contextual,reflection,all_memory,rag,rag_vector \ + --datasets math-hard \ + --n-profiles 200 \ + --n-sessions 30 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --continue-from "$RESULTS_DIR" + +echo "Continue to 30 sessions complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/generate_jobs.sh b/collaborativeagents/slurm/fullscale/generate_jobs.sh new file mode 100644 index 0000000..0bc5c0b --- /dev/null +++ b/collaborativeagents/slurm/fullscale/generate_jobs.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Generate all job scripts (6 methods × 4 profile ranges = 24 jobs) +# Each job: 50 profiles × 15 sessions = 750 sessions ≈ 7-8 hours + +METHODS="vanilla contextual reflection all_memory rag rag_vector" +RANGES="0:50 50:100 100:150 150:200" + +for method in $METHODS; do + for range in $RANGES; do + start=${range%:*} + end=${range#*:} + + cat > run_${method}_p${start}.sh << EOF +#!/bin/bash +#SBATCH --job-name=exp_${method}_p${start} +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_${method}_p${start}_%j.out +#SBATCH --error=exp_${method}_p${start}_%j.err + +# Full run: ${method} method, profiles ${start}-${end} (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="\${PWD}/src:\${PWD}/collaborativeagents:\${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \\ + --model \$MODEL_8B --port 8003 --tensor-parallel-size 1 \\ + --gpu-memory-utilization 0.5 --max-model-len 8192 \\ + --dtype bfloat16 --disable-log-requests & + +for i in \$(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \\ + --methods ${method} \\ + --datasets math-hard \\ + --n-profiles 200 \\ + --start-profile ${start} \\ + --end-profile ${end} \\ + --n-sessions 15 \\ + --max-turns 8 \\ + --use-vllm \\ + --use-openai-user \\ + --openai-user-model gpt-5-mini \\ + --reward-mode llm \\ + --vllm-agent-url http://localhost:8003/v1 \\ + --parallel-profiles 25 \\ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \\ + --output-dir ../results/fullscale_15sess + +echo "${method} p${start}-${end} complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true +EOF + chmod +x run_${method}_p${start}.sh + echo "Created run_${method}_p${start}.sh" + done +done + +echo "" +echo "Generated 24 job scripts (6 methods × 4 profile ranges)" +echo "Each job: 50 profiles × 15 sessions = 750 sessions" +echo "Estimated time per job: ~7-8 hours" diff --git a/collaborativeagents/slurm/fullscale/run_all_memory_p0.sh b/collaborativeagents/slurm/fullscale/run_all_memory_p0.sh new file mode 100755 index 0000000..bb7968b --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_all_memory_p0.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_all_memory_p0 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_all_memory_p0_%j.out +#SBATCH --error=exp_all_memory_p0_%j.err + +# Full run: all_memory method, profiles 0-50 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods all_memory \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 0 \ + --end-profile 50 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "all_memory p0-50 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_all_memory_p100.sh b/collaborativeagents/slurm/fullscale/run_all_memory_p100.sh new file mode 100755 index 0000000..21db6de --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_all_memory_p100.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_all_memory_p100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_all_memory_p100_%j.out +#SBATCH --error=exp_all_memory_p100_%j.err + +# Full run: all_memory method, profiles 100-150 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods all_memory \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 100 \ + --end-profile 150 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "all_memory p100-150 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_all_memory_p150.sh b/collaborativeagents/slurm/fullscale/run_all_memory_p150.sh new file mode 100755 index 0000000..da7a729 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_all_memory_p150.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_all_memory_p150 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_all_memory_p150_%j.out +#SBATCH --error=exp_all_memory_p150_%j.err + +# Full run: all_memory method, profiles 150-200 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods all_memory \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 150 \ + --end-profile 200 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "all_memory p150-200 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_all_memory_p50.sh b/collaborativeagents/slurm/fullscale/run_all_memory_p50.sh new file mode 100755 index 0000000..60bc9ee --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_all_memory_p50.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_all_memory_p50 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_all_memory_p50_%j.out +#SBATCH --error=exp_all_memory_p50_%j.err + +# Full run: all_memory method, profiles 50-100 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods all_memory \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 50 \ + --end-profile 100 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "all_memory p50-100 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_contextual_p0.sh b/collaborativeagents/slurm/fullscale/run_contextual_p0.sh new file mode 100755 index 0000000..6fa0211 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_contextual_p0.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_contextual_p0 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_contextual_p0_%j.out +#SBATCH --error=exp_contextual_p0_%j.err + +# Full run: contextual method, profiles 0-50 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods contextual \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 0 \ + --end-profile 50 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "contextual p0-50 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_contextual_p100.sh b/collaborativeagents/slurm/fullscale/run_contextual_p100.sh new file mode 100755 index 0000000..8250c19 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_contextual_p100.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_contextual_p100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_contextual_p100_%j.out +#SBATCH --error=exp_contextual_p100_%j.err + +# Full run: contextual method, profiles 100-150 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods contextual \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 100 \ + --end-profile 150 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "contextual p100-150 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_contextual_p150.sh b/collaborativeagents/slurm/fullscale/run_contextual_p150.sh new file mode 100755 index 0000000..fb14058 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_contextual_p150.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_contextual_p150 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_contextual_p150_%j.out +#SBATCH --error=exp_contextual_p150_%j.err + +# Full run: contextual method, profiles 150-200 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods contextual \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 150 \ + --end-profile 200 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "contextual p150-200 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_contextual_p50.sh b/collaborativeagents/slurm/fullscale/run_contextual_p50.sh new file mode 100755 index 0000000..8b1788e --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_contextual_p50.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_contextual_p50 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_contextual_p50_%j.out +#SBATCH --error=exp_contextual_p50_%j.err + +# Full run: contextual method, profiles 50-100 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods contextual \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 50 \ + --end-profile 100 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "contextual p50-100 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_p0.sh b/collaborativeagents/slurm/fullscale/run_rag_p0.sh new file mode 100755 index 0000000..de4f038 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_p0.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_p0 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_p0_%j.out +#SBATCH --error=exp_rag_p0_%j.err + +# Full run: rag method, profiles 0-50 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 0 \ + --end-profile 50 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag p0-50 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_p100.sh b/collaborativeagents/slurm/fullscale/run_rag_p100.sh new file mode 100755 index 0000000..c9b9d7e --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_p100.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_p100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_p100_%j.out +#SBATCH --error=exp_rag_p100_%j.err + +# Full run: rag method, profiles 100-150 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 100 \ + --end-profile 150 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag p100-150 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_p150.sh b/collaborativeagents/slurm/fullscale/run_rag_p150.sh new file mode 100755 index 0000000..0ec5e4f --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_p150.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_p150 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_p150_%j.out +#SBATCH --error=exp_rag_p150_%j.err + +# Full run: rag method, profiles 150-200 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 150 \ + --end-profile 200 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag p150-200 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_p50.sh b/collaborativeagents/slurm/fullscale/run_rag_p50.sh new file mode 100755 index 0000000..b625300 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_p50.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_p50 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_p50_%j.out +#SBATCH --error=exp_rag_p50_%j.err + +# Full run: rag method, profiles 50-100 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 50 \ + --end-profile 100 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag p50-100 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_vector_p0.sh b/collaborativeagents/slurm/fullscale/run_rag_vector_p0.sh new file mode 100755 index 0000000..1f28f8f --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_vector_p0.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_vector_p0 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_vector_p0_%j.out +#SBATCH --error=exp_rag_vector_p0_%j.err + +# Full run: rag_vector method, profiles 0-50 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 0 \ + --end-profile 50 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag_vector p0-50 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_vector_p100.sh b/collaborativeagents/slurm/fullscale/run_rag_vector_p100.sh new file mode 100755 index 0000000..b658bab --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_vector_p100.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_vector_p100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_vector_p100_%j.out +#SBATCH --error=exp_rag_vector_p100_%j.err + +# Full run: rag_vector method, profiles 100-150 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 100 \ + --end-profile 150 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag_vector p100-150 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_vector_p150.sh b/collaborativeagents/slurm/fullscale/run_rag_vector_p150.sh new file mode 100755 index 0000000..8c2458f --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_vector_p150.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_vector_p150 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_vector_p150_%j.out +#SBATCH --error=exp_rag_vector_p150_%j.err + +# Full run: rag_vector method, profiles 150-200 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 150 \ + --end-profile 200 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag_vector p150-200 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_rag_vector_p50.sh b/collaborativeagents/slurm/fullscale/run_rag_vector_p50.sh new file mode 100755 index 0000000..afb0164 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_rag_vector_p50.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_rag_vector_p50 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_rag_vector_p50_%j.out +#SBATCH --error=exp_rag_vector_p50_%j.err + +# Full run: rag_vector method, profiles 50-100 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 50 \ + --end-profile 100 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "rag_vector p50-100 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_reflection_p0.sh b/collaborativeagents/slurm/fullscale/run_reflection_p0.sh new file mode 100755 index 0000000..f5d5649 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_reflection_p0.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_reflection_p0 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_reflection_p0_%j.out +#SBATCH --error=exp_reflection_p0_%j.err + +# Full run: reflection method, profiles 0-50 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods reflection \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 0 \ + --end-profile 50 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "reflection p0-50 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_reflection_p100.sh b/collaborativeagents/slurm/fullscale/run_reflection_p100.sh new file mode 100755 index 0000000..68f7047 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_reflection_p100.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_reflection_p100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_reflection_p100_%j.out +#SBATCH --error=exp_reflection_p100_%j.err + +# Full run: reflection method, profiles 100-150 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods reflection \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 100 \ + --end-profile 150 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "reflection p100-150 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_reflection_p150.sh b/collaborativeagents/slurm/fullscale/run_reflection_p150.sh new file mode 100755 index 0000000..a451e49 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_reflection_p150.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_reflection_p150 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_reflection_p150_%j.out +#SBATCH --error=exp_reflection_p150_%j.err + +# Full run: reflection method, profiles 150-200 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods reflection \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 150 \ + --end-profile 200 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "reflection p150-200 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_reflection_p50.sh b/collaborativeagents/slurm/fullscale/run_reflection_p50.sh new file mode 100755 index 0000000..dc977d7 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_reflection_p50.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_reflection_p50 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_reflection_p50_%j.out +#SBATCH --error=exp_reflection_p50_%j.err + +# Full run: reflection method, profiles 50-100 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods reflection \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 50 \ + --end-profile 100 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "reflection p50-100 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_vanilla_p0.sh b/collaborativeagents/slurm/fullscale/run_vanilla_p0.sh new file mode 100755 index 0000000..f5706c8 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_vanilla_p0.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_vanilla_p0 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_vanilla_p0_%j.out +#SBATCH --error=exp_vanilla_p0_%j.err + +# Full run: vanilla method, profiles 0-50 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 0 \ + --end-profile 50 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "vanilla p0-50 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_vanilla_p100.sh b/collaborativeagents/slurm/fullscale/run_vanilla_p100.sh new file mode 100755 index 0000000..8ca9ce1 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_vanilla_p100.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_vanilla_p100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_vanilla_p100_%j.out +#SBATCH --error=exp_vanilla_p100_%j.err + +# Full run: vanilla method, profiles 100-150 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 100 \ + --end-profile 150 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "vanilla p100-150 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_vanilla_p150.sh b/collaborativeagents/slurm/fullscale/run_vanilla_p150.sh new file mode 100755 index 0000000..07ff6d3 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_vanilla_p150.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_vanilla_p150 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_vanilla_p150_%j.out +#SBATCH --error=exp_vanilla_p150_%j.err + +# Full run: vanilla method, profiles 150-200 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 150 \ + --end-profile 200 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "vanilla p150-200 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/run_vanilla_p50.sh b/collaborativeagents/slurm/fullscale/run_vanilla_p50.sh new file mode 100755 index 0000000..d77b881 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/run_vanilla_p50.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=exp_vanilla_p50 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=12:00:00 +#SBATCH --output=exp_vanilla_p50_%j.out +#SBATCH --error=exp_vanilla_p50_%j.err + +# Full run: vanilla method, profiles 50-100 (50 profiles × 15 sessions) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 200 \ + --start-profile 50 \ + --end-profile 100 \ + --n-sessions 15 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_15sess + +echo "vanilla p50-100 complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/submit_all.sh b/collaborativeagents/slurm/fullscale/submit_all.sh new file mode 100644 index 0000000..5b76169 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/submit_all.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Submit all 24 jobs for full-scale experiment +# Total: 200 profiles × 6 methods × 15 sessions = 18,000 sessions +# Split: 6 methods × 4 profile ranges (50 each) = 24 jobs +# Per job: 50 profiles × 15 sessions = 750 sessions ≈ 7-8 hours + +echo "Submitting all 24 jobs for full-scale experiment..." +echo "Total: 200 profiles × 6 methods × 15 sessions = 18,000 sessions" +echo "Split: 24 jobs (6 methods × 4 profile ranges of 50)" +echo "" + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/slurm/fullscale + +JOBS="" + +for script in run_*.sh; do + if [[ "$script" != "generate_jobs.sh" && "$script" != "submit_all.sh" && "$script" != "test_run.sh" ]]; then + JOB_ID=$(sbatch "$script" | awk '{print $4}') + JOBS="$JOBS $JOB_ID" + echo "Submitted $script -> Job ID: $JOB_ID" + fi +done + +echo "" +echo "All jobs submitted!" +echo "Job IDs:$JOBS" +echo "" +echo "Monitor with: squeue -u \$USER" +echo "Check results in: collaborativeagents/results/fullscale_15sess/" diff --git a/collaborativeagents/slurm/fullscale/test_25parallel.sh b/collaborativeagents/slurm/fullscale/test_25parallel.sh new file mode 100644 index 0000000..09d5ddb --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_25parallel.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=test_25parallel +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=1:00:00 +#SBATCH --output=test_25parallel_%j.out +#SBATCH --error=test_25parallel_%j.err + +# Quick test: 25 profiles × 2 sessions × 1 method (vanilla) = 50 sessions +# With 25 parallel profiles to measure realistic throughput + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +# Test with 25 parallel profiles +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 25 \ + --n-sessions 2 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 25 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/test_25parallel_$(date +%Y%m%d_%H%M%S) + +echo "Test complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/test_25parallel_15649074.err b/collaborativeagents/slurm/fullscale/test_25parallel_15649074.err new file mode 100644 index 0000000..96ed829 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_25parallel_15649074.err @@ -0,0 +1,386 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:08<00:26, 8.77s/it] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:10<00:09, 4.67s/it] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:19<00:06, 6.61s/it] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:27<00:00, 7.00s/it] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:27<00:00, 6.77s/it] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 14.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:03, 15.08it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 16.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 16.68it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 18.08it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 18.87it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.61it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.61it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.26it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:01, 20.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 21.35it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 22.13it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 22.75it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:01<00:00, 23.31it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 23.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 23.75it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 24.35it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 21.03it/s] +[0;36m(EngineCore_DP0 pid=2749050)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:04, 8.11it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:01, 16.75it/s]
Capturing CUDA graphs (decode, FULL): 20%|██ | 7/35 [00:00<00:01, 19.48it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:01, 20.84it/s]
Capturing CUDA graphs (decode, FULL): 37%|███▋ | 13/35 [00:00<00:01, 21.74it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:00<00:00, 22.44it/s]
Capturing CUDA graphs (decode, FULL): 54%|█████▍ | 19/35 [00:00<00:00, 22.82it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:01<00:00, 23.12it/s]
Capturing CUDA graphs (decode, FULL): 71%|███████▏ | 25/35 [00:01<00:00, 23.29it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:01<00:00, 23.60it/s]
Capturing CUDA graphs (decode, FULL): 89%|████████▊ | 31/35 [00:01<00:00, 23.89it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:01<00:00, 24.07it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 22.39it/s] +[0;36m(APIServer pid=2748545)[0;0m INFO: Started server process [2748545] +[0;36m(APIServer pid=2748545)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=2748545)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-26 09:27:47,068 - INFO - Loaded dataset: math-hard +2026-01-26 09:27:47,068 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 09:27:47,089 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 09:27:47,090 - INFO - Running method: vanilla +2026-01-26 09:27:47,092 - INFO - Running profiles 0 to 24 (25 remaining) +2026-01-26 09:27:47,092 - INFO - Using BATCH processing (async OpenAI user) for vanilla +2026-01-26 09:27:51,946 - INFO - Batch: 25 profiles, 50 sessions remaining +2026-01-26 09:28:12,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:12,493 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:13,059 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:14,056 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:14,096 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:16,728 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:17,048 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:17,207 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:17,852 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:17,921 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:18,025 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:18,315 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:18,717 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:18,721 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:19,139 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:19,426 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:19,559 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:19,669 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:20,116 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:20,490 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:21,361 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:22,541 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:26,115 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:27,029 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:28:30,969 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:28,992 - INFO - Retrying request to /chat/completions in 0.497164 seconds +2026-01-26 09:30:49,151 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:51,550 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:51,602 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:52,002 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:52,304 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:53,195 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:55,825 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:56,818 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:57,147 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:59,974 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:02,648 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:02,990 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:04,612 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:04,820 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:05,756 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:06,486 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:08,186 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:09,519 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:14,224 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:14,961 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:16,471 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:21,750 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:25,200 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:25,420 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:26,474 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:32:17,226 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:34:46,144 - INFO - Retrying request to /chat/completions in 0.408735 seconds +2026-01-26 09:35:08,723 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:08,941 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:09,311 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:10,222 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:11,410 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:14,879 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:16,671 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:16,957 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:17,153 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:17,391 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:19,785 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:20,686 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:21,933 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:22,094 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:23,911 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:24,217 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:27,229 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:28,836 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:30,822 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:33,169 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:34,386 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:39,633 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:40,925 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:06,800 - INFO - Retrying request to /chat/completions in 0.380454 seconds +2026-01-26 09:38:24,263 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:25,450 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:27,291 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:28,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:29,229 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:29,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:29,517 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:29,743 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:30,599 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:31,944 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:34,329 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:34,755 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:37,718 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:38,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:39,658 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:41,005 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:43,610 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:44,813 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:50,989 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:38:54,975 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:39:00,332 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:39:01,476 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:24,167 - INFO - Retrying request to /chat/completions in 0.497047 seconds +2026-01-26 09:41:40,878 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:41,762 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:42,741 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:43,847 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:44,016 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:44,972 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:45,498 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:47,265 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:48,304 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:49,325 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:51,122 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:51,989 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:52,039 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:52,795 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:53,205 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:55,942 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:56,227 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:57,168 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:41:57,981 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:00,909 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:04,535 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:20,690 - INFO - Retrying request to /chat/completions in 0.488344 seconds +2026-01-26 09:44:35,415 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:37,095 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:41,452 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:42,469 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:42,474 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:42,828 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:44,768 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:44,896 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:46,075 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:48,796 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:49,016 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:50,272 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:50,282 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:51,365 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:52,454 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:54,527 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:55,603 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:44:56,960 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:45:04,617 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:19,945 - INFO - Retrying request to /chat/completions in 0.436698 seconds +2026-01-26 09:47:35,313 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:37,522 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:39,093 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:39,424 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:39,624 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:40,126 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:40,752 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:40,903 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:42,081 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:43,785 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:44,603 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:44,614 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:46,264 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:52,539 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:53,004 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:56,231 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:47:57,495 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:06,564 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:20,835 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:37,276 - INFO - Retrying request to /chat/completions in 0.452468 seconds +2026-01-26 09:50:51,899 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:53,721 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:55,014 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:55,582 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:55,590 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:59,402 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:59,582 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:50:59,882 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:00,612 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:01,288 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:02,592 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:02,770 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:03,504 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:03,723 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:04,449 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:04,767 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:05,280 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:06,810 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:51:34,003 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:53:51,601 - INFO - Session round 1/2: 25 total, 58 sessions/hr +2026-01-26 09:53:51,604 - INFO - Retrying request to /chat/completions in 0.444179 seconds +2026-01-26 09:54:06,889 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:11,520 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:12,150 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:12,415 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:12,433 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:13,872 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:14,604 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:15,711 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:15,926 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:16,676 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:16,752 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:20,312 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:20,851 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:22,383 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:23,091 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:23,274 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:23,448 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:23,779 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:24,746 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:24,908 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:25,362 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:25,671 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:38,199 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:43,115 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:45,969 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:05,543 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:56:42,149 - INFO - Retrying request to /chat/completions in 0.419778 seconds +2026-01-26 09:56:57,132 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:02,101 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:02,138 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:06,250 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:09,388 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:10,027 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:11,156 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:11,977 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:12,355 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:13,457 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:13,470 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:13,663 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:14,452 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:15,062 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:18,548 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:20,788 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:23,961 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:31,290 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:32,055 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:33,034 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:35,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:36,838 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:39,944 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:40,056 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:40,920 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:57:59,606 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:58:27,720 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:58:33,775 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:58:34,589 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:46,860 - INFO - Retrying request to /chat/completions in 0.416438 seconds +2026-01-26 10:01:09,640 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:10,750 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:14,267 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:15,896 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:21,140 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:21,415 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:22,737 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:27,180 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:29,997 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:30,388 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:30,773 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:34,505 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:34,894 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:37,212 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:39,069 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:39,363 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:43,345 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:43,923 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:46,439 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:47,877 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:51,629 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:02:02,535 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:02:14,872 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:02:15,377 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:02:16,444 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:02:38,111 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:03:35,818 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:05:41,154 - INFO - Retrying request to /chat/completions in 0.399071 seconds +2026-01-26 10:05:57,816 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:02,020 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:04,719 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:06,046 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:08,946 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:10,046 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:10,720 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:10,873 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:13,979 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:15,928 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:16,923 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:17,044 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:23,932 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:36,979 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:37,422 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:42,713 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:42,728 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:42,734 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:44,033 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:44,136 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:12,836 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:13,878 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:14,207 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:36,442 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:59,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:08:22,397 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:11,478 - INFO - Retrying request to /chat/completions in 0.409762 seconds +2026-01-26 10:10:35,080 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:35,410 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:35,818 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:36,280 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:37,382 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:38,684 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:41,309 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:42,476 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:43,322 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:10:55,627 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:04,211 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:04,302 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:04,372 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:05,028 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:18,536 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:18,718 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:42,576 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:11:52,841 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:01,265 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:02,718 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:51,250 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:16,124 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:15:06,339 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:16:45,339 - INFO - Retrying request to /chat/completions in 0.479317 seconds +2026-01-26 10:17:06,803 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:07,949 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:10,203 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:10,510 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:11,396 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:11,643 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:15,085 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:15,376 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:15,488 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:36,253 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:37,970 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:37,998 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:17:40,137 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:13,980 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:47,227 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:16,033 - INFO - Retrying request to /chat/completions in 0.385396 seconds +2026-01-26 10:20:41,346 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:43,646 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:43,738 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:43,752 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:44,201 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:48,133 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:49,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:53,768 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:57,170 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:21:01,116 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:21:11,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:21:29,714 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:22:19,469 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:22:24,392 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:23:53,532 - INFO - Retrying request to /chat/completions in 0.431884 seconds +2026-01-26 10:24:16,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:17,281 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:20,169 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:22,116 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:25,715 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:26,553 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:28,568 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:33,226 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:40,874 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:24:41,777 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +[2026-01-26T10:24:42.210] error: *** JOB 15649074 ON gpua050 CANCELLED AT 2026-01-26T10:24:42 DUE TO TIME LIMIT *** diff --git a/collaborativeagents/slurm/fullscale/test_50parallel.sh b/collaborativeagents/slurm/fullscale/test_50parallel.sh new file mode 100644 index 0000000..0a48e2d --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_50parallel.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=test_50parallel +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=1:00:00 +#SBATCH --output=test_50parallel_%j.out +#SBATCH --error=test_50parallel_%j.err + +# Test: 50 profiles × 2 sessions = 100 sessions with 50 parallel profiles +# Compare throughput vs 25 parallel + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# Increase GPU utilization to 60% for higher throughput +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.6 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 50 \ + --n-sessions 2 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 50 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/test_50parallel_$(date +%Y%m%d_%H%M%S) + +echo "Test complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/test_50parallel_15649149.err b/collaborativeagents/slurm/fullscale/test_50parallel_15649149.err new file mode 100644 index 0000000..358fd24 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_50parallel_15649149.err @@ -0,0 +1,504 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:05<00:17, 5.94s/it] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:07<00:06, 3.30s/it] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:13<00:04, 4.70s/it] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:18<00:00, 4.88s/it] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:18<00:00, 4.73s/it] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 14.62it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:03, 14.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 16.03it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 16.65it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 17.98it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 18.83it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.62it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.64it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.15it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.74it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:01, 20.32it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 21.27it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 22.12it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 22.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:01<00:00, 23.41it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 24.07it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 24.00it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 24.68it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 21.04it/s] +[0;36m(EngineCore_DP0 pid=2003864)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:04, 7.74it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:01, 16.62it/s]
Capturing CUDA graphs (decode, FULL): 20%|██ | 7/35 [00:00<00:01, 19.54it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:01, 21.02it/s]
Capturing CUDA graphs (decode, FULL): 37%|███▋ | 13/35 [00:00<00:00, 22.01it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:00<00:00, 22.69it/s]
Capturing CUDA graphs (decode, FULL): 54%|█████▍ | 19/35 [00:00<00:00, 23.12it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:01<00:00, 23.45it/s]
Capturing CUDA graphs (decode, FULL): 71%|███████▏ | 25/35 [00:01<00:00, 23.73it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:01<00:00, 24.03it/s]
Capturing CUDA graphs (decode, FULL): 89%|████████▊ | 31/35 [00:01<00:00, 24.21it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:01<00:00, 24.31it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 22.61it/s] +[0;36m(APIServer pid=2003364)[0;0m INFO: Started server process [2003364] +[0;36m(APIServer pid=2003364)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=2003364)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-26 09:29:43,637 - INFO - Loaded dataset: math-hard +2026-01-26 09:29:43,637 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 09:29:43,658 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 09:29:43,659 - INFO - Running method: vanilla +2026-01-26 09:29:43,660 - INFO - Running profiles 0 to 49 (50 remaining) +2026-01-26 09:29:43,660 - INFO - Using BATCH processing (async OpenAI user) for vanilla +2026-01-26 09:29:46,941 - INFO - Batch: 50 profiles, 100 sessions remaining +2026-01-26 09:30:06,454 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:08,319 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:08,451 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:08,842 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:09,445 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:09,759 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:10,606 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:10,842 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:13,059 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:13,117 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:13,149 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:13,240 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:13,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:13,823 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:14,114 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:14,450 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:16,522 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:17,201 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:17,277 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:17,616 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:18,027 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:18,492 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:20,496 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:21,021 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:21,193 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:21,371 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:21,510 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:21,833 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:24,294 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:27,007 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:32,273 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:34,151 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:34,690 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:34,758 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:35,910 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:36,850 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:37,630 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:40,618 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:41,190 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:42,393 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:42,786 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:44,872 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:46,049 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:46,231 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:46,297 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:47,229 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:48,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:48,781 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:52,819 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:30:57,372 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:31:13,018 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:34,869 - INFO - Retrying request to /chat/completions in 0.439588 seconds +2026-01-26 09:35:57,466 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:59,282 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:35:59,318 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:00,650 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:00,770 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:01,826 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:04,364 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:05,040 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:06,043 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:06,251 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:06,340 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:06,460 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:06,777 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:06,784 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:07,939 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:09,820 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:11,725 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:11,939 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:13,530 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:13,664 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:14,413 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:14,652 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:16,100 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:16,352 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:16,839 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:18,001 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:18,441 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:21,080 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:24,896 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:25,097 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:26,581 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:26,911 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:27,043 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:28,834 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:29,102 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:30,211 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:30,514 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:32,879 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:34,460 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:37,520 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:38,116 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:39,596 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:41,792 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:44,867 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:45,135 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:45,736 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:51,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:55,416 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:57,749 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:36:59,238 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:13,030 - INFO - Retrying request to /chat/completions in 0.400120 seconds +2026-01-26 09:42:30,585 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:30,892 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:31,450 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:33,206 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:35,779 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:36,014 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:36,083 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:36,327 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:36,398 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:36,620 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:36,796 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:37,385 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:37,527 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:38,422 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:40,279 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:40,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:41,680 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:42,121 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:42,257 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:43,928 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:43,971 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:45,107 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:46,582 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:46,663 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:47,554 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:47,871 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:49,446 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:49,556 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:51,016 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:51,895 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:52,636 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:52,757 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:53,376 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:53,935 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:42:59,986 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:01,144 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:03,484 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:04,528 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:04,605 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:05,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:05,330 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:05,565 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:06,122 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:06,168 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:11,286 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:14,390 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:18,542 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:20,316 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:43:30,080 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:28,792 - INFO - Retrying request to /chat/completions in 0.405667 seconds +2026-01-26 09:48:45,809 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:48,709 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:48,991 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:50,876 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:50,995 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:52,823 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:52,903 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:52,959 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:53,382 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:53,445 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:53,557 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:53,729 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:53,902 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:54,511 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:56,151 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:56,347 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:48:59,690 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:00,321 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:01,044 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:01,755 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:02,910 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:04,012 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:05,573 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:05,928 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:07,146 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:07,622 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:10,257 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:10,921 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:12,097 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:12,780 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:13,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:14,368 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:14,869 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:15,727 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:15,874 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:16,892 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:18,076 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:18,132 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:18,751 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:19,863 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:20,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:21,057 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:23,432 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:26,466 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:31,006 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:49:49,558 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:36,689 - INFO - Retrying request to /chat/completions in 0.430394 seconds +2026-01-26 09:54:54,293 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:54,487 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:54,829 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:56,069 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:57,256 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:57,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:57,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:57,650 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:57,799 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:58,401 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:54:58,714 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:00,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:01,298 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:01,735 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:04,111 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:04,116 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:04,320 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:05,903 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:05,972 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:07,051 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:07,430 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:07,842 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:08,243 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:09,065 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:11,827 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:11,834 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:12,691 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:14,225 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:16,456 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:16,543 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:18,805 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:20,407 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:20,887 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:22,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:23,408 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:23,884 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:23,986 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:24,368 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:25,977 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:26,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:26,887 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:27,193 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:28,219 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:29,328 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 09:55:52,261 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:21,589 - INFO - Retrying request to /chat/completions in 0.397408 seconds +2026-01-26 10:00:36,291 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:36,955 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:37,823 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:38,707 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:39,517 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:41,582 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:42,114 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:42,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:42,721 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:42,969 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:42,988 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:43,058 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:43,342 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:44,557 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:44,657 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:45,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:45,269 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:45,509 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:45,799 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:45,801 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:46,105 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:47,159 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:47,427 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:47,816 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:48,604 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:49,689 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:49,739 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:51,089 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:52,928 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:54,907 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:55,184 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:56,598 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:59,118 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:00:59,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:02,545 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:03,044 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:03,338 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:06,460 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:08,416 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:10,499 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:12,694 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:15,341 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:01:43,942 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:17,314 - INFO - Retrying request to /chat/completions in 0.385894 seconds +2026-01-26 10:06:32,587 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:33,300 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:37,112 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:38,036 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:38,408 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:38,484 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:38,507 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:40,072 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:40,536 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:41,018 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:41,092 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:41,686 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:41,761 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:41,945 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:42,008 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:43,444 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:44,347 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:44,820 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:44,906 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:45,643 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:45,780 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:45,909 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:47,560 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:47,605 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:48,560 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:50,804 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:54,243 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:54,294 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:57,079 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:57,277 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:58,041 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:06:58,423 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:01,704 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:06,037 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:07,673 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:12,023 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:16,183 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:20,226 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:22,800 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:26,791 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:07:54,018 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:22,041 - INFO - Retrying request to /chat/completions in 0.454918 seconds +2026-01-26 10:12:36,789 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:40,297 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:41,629 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:42,714 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:42,757 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:44,028 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:45,344 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:46,438 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:46,878 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:47,563 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:47,805 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:48,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:49,145 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:49,361 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:49,564 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:49,825 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:50,085 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:51,229 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:51,874 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:51,975 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:53,724 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:53,785 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:54,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:54,266 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:54,808 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:55,152 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:56,596 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:12:57,495 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:00,949 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:02,102 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:04,013 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:05,406 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:07,431 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:08,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:09,641 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:13,580 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:14,384 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:15,505 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:13:28,509 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:02,217 - INFO - Session round 1/2: 50 total, 62 sessions/hr +2026-01-26 10:18:02,221 - INFO - Retrying request to /chat/completions in 0.390137 seconds +2026-01-26 10:18:23,953 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:25,452 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:25,533 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:26,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:26,808 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:26,971 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:27,348 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:27,652 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:28,804 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:28,808 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:29,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:30,377 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:31,132 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:31,405 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:31,644 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:31,698 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:31,953 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:32,283 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:32,636 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:32,842 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:32,988 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:33,279 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:35,981 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:36,043 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:36,225 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:40,684 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:41,084 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:42,000 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:42,697 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:48,873 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:49,383 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:50,056 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:51,900 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:52,030 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:52,995 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:54,433 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:54,917 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:56,763 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:18:58,988 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:00,456 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:01,784 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:02,251 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:03,579 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:03,788 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:08,994 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:10,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:10,942 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:18,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:23,243 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:19:25,144 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:20:57,966 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:21:27,035 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:12,495 - INFO - Retrying request to /chat/completions in 0.494117 seconds +2026-01-26 10:25:29,226 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:32,090 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:36,992 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:37,517 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:37,586 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:37,687 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:38,600 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:39,260 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:39,436 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:41,983 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:42,064 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:42,731 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:43,529 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:44,137 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:44,812 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:44,870 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:45,136 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:45,455 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:46,477 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:48,792 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:49,168 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:51,257 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:51,762 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:53,145 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:58,678 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:25:59,306 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:01,408 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:03,426 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:04,074 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:04,416 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:05,245 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:07,119 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:07,177 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:07,274 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:08,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:08,985 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:09,589 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:09,620 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:10,369 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:13,493 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:14,494 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:18,082 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:19,637 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:19,726 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:27,344 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:27,496 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:30,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:35,051 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:35,315 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:37,814 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:41,709 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:26:42,540 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +[2026-01-26T10:26:43.011] error: *** JOB 15649149 ON gpua019 CANCELLED AT 2026-01-26T10:26:43 DUE TO TIME LIMIT *** diff --git a/collaborativeagents/slurm/fullscale/test_batch_fix.sh b/collaborativeagents/slurm/fullscale/test_batch_fix.sh new file mode 100644 index 0000000..ae87d99 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_batch_fix.sh @@ -0,0 +1,69 @@ +#!/bin/bash +#SBATCH --job-name=test_batch_fix +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=0:30:00 +#SBATCH --output=test_batch_fix_%j.out +#SBATCH --error=test_batch_fix_%j.err + +# Quick test: Verify batched agent calls fix +# 10 profiles × 2 sessions = 20 sessions with 10 parallel +# Should see ~500+ sessions/hr with batching (vs ~60 without) + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# Start vLLM server +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts + +# Test with vanilla (simplest method) +echo "=== Testing batched agent calls ===" +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 10 \ + --n-sessions 2 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 10 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/test_batch_fix_$(date +%Y%m%d_%H%M%S) + +echo "Test complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/test_batch_fix_15651956.err b/collaborativeagents/slurm/fullscale/test_batch_fix_15651956.err new file mode 100644 index 0000000..a7574bf --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_batch_fix_15651956.err @@ -0,0 +1,165 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:06<00:18, 6.31s/it] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:08<00:07, 3.60s/it] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:15<00:05, 5.44s/it] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:21<00:00, 5.61s/it] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:21<00:00, 5.37s/it] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 14.78it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:03, 15.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 16.09it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 16.68it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 18.13it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 18.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.71it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.71it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.34it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.87it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:00, 22.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 22.48it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 22.97it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 23.36it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:01<00:00, 23.76it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 24.28it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 24.65it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 25.04it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 21.70it/s] +[0;36m(EngineCore_DP0 pid=2779888)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 6%|▌ | 2/35 [00:00<00:01, 19.85it/s]
Capturing CUDA graphs (decode, FULL): 14%|█▍ | 5/35 [00:00<00:01, 21.79it/s]
Capturing CUDA graphs (decode, FULL): 23%|██▎ | 8/35 [00:00<00:01, 22.38it/s]
Capturing CUDA graphs (decode, FULL): 31%|███▏ | 11/35 [00:00<00:01, 22.76it/s]
Capturing CUDA graphs (decode, FULL): 40%|████ | 14/35 [00:00<00:00, 23.18it/s]
Capturing CUDA graphs (decode, FULL): 49%|████▊ | 17/35 [00:00<00:00, 23.43it/s]
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 20/35 [00:00<00:00, 23.60it/s]
Capturing CUDA graphs (decode, FULL): 66%|██████▌ | 23/35 [00:00<00:00, 23.69it/s]
Capturing CUDA graphs (decode, FULL): 74%|███████▍ | 26/35 [00:01<00:00, 23.90it/s]
Capturing CUDA graphs (decode, FULL): 83%|████████▎ | 29/35 [00:01<00:00, 24.13it/s]
Capturing CUDA graphs (decode, FULL): 91%|█████████▏| 32/35 [00:01<00:00, 24.33it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 24.53it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 23.67it/s] +[0;36m(APIServer pid=2779820)[0;0m INFO: Started server process [2779820] +[0;36m(APIServer pid=2779820)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=2779820)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-26 10:49:25,310 - INFO - Loaded dataset: math-hard +2026-01-26 10:49:25,310 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 10:49:25,331 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 10:49:25,332 - INFO - Running method: vanilla +2026-01-26 10:49:25,333 - INFO - Running profiles 0 to 9 (10 remaining) +2026-01-26 10:49:25,333 - INFO - Using BATCH processing (async OpenAI user) for vanilla +2026-01-26 10:49:27,629 - INFO - Batch: 10 profiles, 20 sessions remaining +2026-01-26 10:49:51,327 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:49:54,771 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:49:55,968 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:49:55,971 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:49:56,111 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:49:58,494 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:50:00,365 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:50:00,481 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:50:01,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:50:02,557 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:50:31,911 - INFO - Retrying request to /chat/completions in 0.472652 seconds +2026-01-26 10:50:55,766 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:00,321 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:01,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:02,348 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:03,035 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:07,706 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:10,831 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:17,046 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:20,228 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:51:29,972 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:52:18,950 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:52:48,948 - INFO - Retrying request to /chat/completions in 0.401385 seconds +2026-01-26 10:53:08,512 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:17,427 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:17,958 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:20,633 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:23,267 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:24,807 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:25,462 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:39,454 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:53:45,026 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:54:42,712 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:14,191 - INFO - Retrying request to /chat/completions in 0.465460 seconds +2026-01-26 10:55:30,981 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:36,802 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:36,998 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:37,952 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:42,469 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:46,764 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:48,435 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:55,025 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:55:56,037 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:16,243 - INFO - Retrying request to /chat/completions in 0.475638 seconds +2026-01-26 10:56:33,092 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:38,487 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:43,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:44,094 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:44,184 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:48,210 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:52,409 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:56:58,024 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:57:03,714 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:57:17,816 - INFO - Retrying request to /chat/completions in 0.475195 seconds +2026-01-26 10:57:38,360 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:57:43,248 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:57:44,051 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:57:53,040 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:58:04,492 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:58:09,427 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:58:22,280 - INFO - Retrying request to /chat/completions in 0.427496 seconds +2026-01-26 10:58:48,028 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:58:50,995 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:59:05,351 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:59:19,341 - INFO - Retrying request to /chat/completions in 0.401335 seconds +2026-01-26 10:59:37,858 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:59:43,064 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 10:59:52,406 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:03,266 - INFO - Session round 1/2: 10 total, 56 sessions/hr +2026-01-26 11:00:03,268 - INFO - Retrying request to /chat/completions in 0.441368 seconds +2026-01-26 11:00:23,708 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:26,029 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:26,087 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:27,850 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:28,241 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:30,520 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:31,723 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:32,826 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:37,576 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:39,479 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:00:47,699 - INFO - Retrying request to /chat/completions in 0.433057 seconds +2026-01-26 11:01:08,417 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:11,013 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:12,133 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:12,170 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:23,969 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:27,863 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:29,426 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:48,623 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:49,104 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:01:54,091 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:02:31,426 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:02:46,245 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:02:57,910 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:03:06,851 - INFO - Retrying request to /chat/completions in 0.429348 seconds +2026-01-26 11:03:38,900 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:03:40,960 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:03:42,031 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:04:11,946 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:04:12,383 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:04:15,523 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:04:16,260 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:04:18,787 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:04:19,674 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:05:49,055 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:00,150 - INFO - Retrying request to /chat/completions in 0.448308 seconds +2026-01-26 11:06:35,563 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:44,567 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:44,848 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:52,764 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:54,443 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:54,793 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:06:56,054 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:08:54,799 - INFO - Retrying request to /chat/completions in 0.489046 seconds +2026-01-26 11:09:48,962 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:09:59,678 - INFO - Retrying request to /chat/completions in 0.414219 seconds +2026-01-26 11:10:23,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:10:23,684 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:10:34,012 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:10:38,223 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:10:41,674 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:10:59,720 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:12:50,321 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:14:34,418 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:14:45,079 - INFO - Retrying request to /chat/completions in 0.408414 seconds +2026-01-26 11:15:06,196 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:15:16,148 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:15:37,062 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:15:37,073 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:15:48,411 - INFO - Retrying request to /chat/completions in 0.397816 seconds +2026-01-26 11:16:22,670 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:16:32,537 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:16:34,381 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:16:54,692 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:17:06,402 - INFO - Retrying request to /chat/completions in 0.418484 seconds +2026-01-26 11:17:25,760 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:17:50,338 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:17:57,400 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-26 11:18:01,918 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +[2026-01-26T11:18:21.428] error: *** JOB 15651956 ON gpua050 CANCELLED AT 2026-01-26T11:18:21 DUE TO TIME LIMIT *** diff --git a/collaborativeagents/slurm/fullscale/test_local_user.sh b/collaborativeagents/slurm/fullscale/test_local_user.sh new file mode 100644 index 0000000..8374a93 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_local_user.sh @@ -0,0 +1,94 @@ +#!/bin/bash +#SBATCH --job-name=test_local_user +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=1:00:00 +#SBATCH --output=test_local_user_%j.out +#SBATCH --error=test_local_user_%j.err + +# Test with LOCAL vLLM user simulator (70B AWQ) instead of OpenAI +# Expected: ~2000+ sessions/hr (vs ~60 with OpenAI API latency) +# +# GPU Layout: +# GPU 0-1: 70B user simulator (AWQ INT4, TP=2) +# GPU 2: 8B agent (~24GB) +# GPU 3: Available for embedding/reranker if needed + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a +source .env +set +a + +# Models +MODEL_70B="hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +# Kill any existing vLLM servers +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 3 + +echo "=== Starting 70B User Simulator (GPU 0-1, TP=2) ===" +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B \ + --port 8004 \ + --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.90 \ + --max-model-len 4096 \ + --quantization awq \ + --dtype float16 \ + --disable-log-requests \ + --guided-decoding-backend outlines & + +echo "=== Starting 8B Agent (GPU 2) ===" +CUDA_VISIBLE_DEVICES=2 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B \ + --port 8003 \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.50 \ + --max-model-len 8192 \ + --dtype bfloat16 \ + --disable-log-requests & + +# Wait for both servers +echo "Waiting for vLLM servers..." +for port in 8004 8003; do + for i in $(seq 1 120); do + curl -s http://localhost:$port/health > /dev/null 2>&1 && break + sleep 2 + done + echo " Port $port ready." +done + +cd collaborativeagents/scripts + +echo "" +echo "=== Running Test: 10 profiles × 2 sessions with LOCAL user simulator ===" +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 10 \ + --n-sessions 2 \ + --max-turns 8 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --reward-mode llm \ + --parallel-profiles 10 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/test_local_user_$(date +%Y%m%d_%H%M%S) + +echo "" +echo "=== Test Complete ===" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/fullscale/test_local_user_15652698.err b/collaborativeagents/slurm/fullscale/test_local_user_15652698.err new file mode 100644 index 0000000..4acc458 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_local_user_15652698.err @@ -0,0 +1,215 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +usage: api_server.py [-h] [--headless] [--api-server-count API_SERVER_COUNT] + [--config CONFIG] [--host HOST] [--port PORT] [--uds UDS] + [--uvicorn-log-level {critical,debug,error,info,trace,warning}] + [--disable-uvicorn-access-log | --no-disable-uvicorn-access-log] + [--allow-credentials | --no-allow-credentials] + [--allowed-origins ALLOWED_ORIGINS] + [--allowed-methods ALLOWED_METHODS] + [--allowed-headers ALLOWED_HEADERS] + [--api-key API_KEY [API_KEY ...]] + [--lora-modules LORA_MODULES [LORA_MODULES ...]] + [--chat-template CHAT_TEMPLATE] + [--chat-template-content-format {auto,openai,string}] + [--trust-request-chat-template | --no-trust-request-chat-template] + [--response-role RESPONSE_ROLE] + [--ssl-keyfile SSL_KEYFILE] [--ssl-certfile SSL_CERTFILE] + [--ssl-ca-certs SSL_CA_CERTS] + [--enable-ssl-refresh | --no-enable-ssl-refresh] + [--ssl-cert-reqs SSL_CERT_REQS] [--root-path ROOT_PATH] + [--middleware MIDDLEWARE] + [--return-tokens-as-token-ids | --no-return-tokens-as-token-ids] + [--disable-frontend-multiprocessing | --no-disable-frontend-multiprocessing] + [--enable-request-id-headers | --no-enable-request-id-headers] + [--enable-auto-tool-choice | --no-enable-auto-tool-choice] + [--exclude-tools-when-tool-choice-none | --no-exclude-tools-when-tool-choice-none] + [--tool-call-parser {deepseek_v3,deepseek_v31,deepseek_v32,ernie45,gigachat3,glm45,granite,granite-20b-fc,hermes,hunyuan_a13b,internlm,jamba,kimi_k2,llama3_json,llama4_json,llama4_pythonic,longcat,minimax,minimax_m2,mistral,olmo3,openai,phi4_mini_json,pythonic,qwen3_coder,qwen3_xml,seed_oss,step3,xlam} or name registered in --tool-parser-plugin] + [--tool-parser-plugin TOOL_PARSER_PLUGIN] + [--tool-server TOOL_SERVER] + [--log-config-file LOG_CONFIG_FILE] + [--max-log-len MAX_LOG_LEN] + [--disable-fastapi-docs | --no-disable-fastapi-docs] + [--enable-prompt-tokens-details | --no-enable-prompt-tokens-details] + [--enable-server-load-tracking | --no-enable-server-load-tracking] + [--enable-force-include-usage | --no-enable-force-include-usage] + [--enable-tokenizer-info-endpoint | --no-enable-tokenizer-info-endpoint] + [--enable-log-outputs | --no-enable-log-outputs] + [--h11-max-incomplete-event-size H11_MAX_INCOMPLETE_EVENT_SIZE] + [--h11-max-header-count H11_MAX_HEADER_COUNT] + [--log-error-stack | --no-log-error-stack] + [--tokens-only | --no-tokens-only] [--model MODEL] + [--runner {auto,draft,generate,pooling}] + [--convert {auto,classify,embed,none,reward}] + [--tokenizer TOKENIZER] + [--tokenizer-mode ['auto', 'deepseek_v32', 'hf', 'mistral', 'slow']] + [--trust-remote-code | --no-trust-remote-code] + [--dtype {auto,bfloat16,float,float16,float32,half}] + [--seed SEED] [--hf-config-path HF_CONFIG_PATH] + [--allowed-local-media-path ALLOWED_LOCAL_MEDIA_PATH] + [--allowed-media-domains ALLOWED_MEDIA_DOMAINS [ALLOWED_MEDIA_DOMAINS ...]] + [--revision REVISION] [--code-revision CODE_REVISION] + [--tokenizer-revision TOKENIZER_REVISION] + [--max-model-len MAX_MODEL_LEN] + [--quantization QUANTIZATION] + [--enforce-eager | --no-enforce-eager] + [--max-logprobs MAX_LOGPROBS] + [--logprobs-mode {processed_logits,processed_logprobs,raw_logits,raw_logprobs}] + [--disable-sliding-window | --no-disable-sliding-window] + [--disable-cascade-attn | --no-disable-cascade-attn] + [--skip-tokenizer-init | --no-skip-tokenizer-init] + [--enable-prompt-embeds | --no-enable-prompt-embeds] + [--served-model-name SERVED_MODEL_NAME [SERVED_MODEL_NAME ...]] + [--config-format ['auto', 'hf', 'mistral']] + [--hf-token [HF_TOKEN]] [--hf-overrides HF_OVERRIDES] + [--pooler-config POOLER_CONFIG] + [--logits-processor-pattern LOGITS_PROCESSOR_PATTERN] + [--generation-config GENERATION_CONFIG] + [--override-generation-config OVERRIDE_GENERATION_CONFIG] + [--enable-sleep-mode | --no-enable-sleep-mode] + [--model-impl ['auto', 'terratorch', 'transformers', 'vllm']] + [--override-attention-dtype OVERRIDE_ATTENTION_DTYPE] + [--logits-processors LOGITS_PROCESSORS [LOGITS_PROCESSORS ...]] + [--io-processor-plugin IO_PROCESSOR_PLUGIN] + [--load-format LOAD_FORMAT] [--download-dir DOWNLOAD_DIR] + [--safetensors-load-strategy SAFETENSORS_LOAD_STRATEGY] + [--model-loader-extra-config MODEL_LOADER_EXTRA_CONFIG] + [--ignore-patterns IGNORE_PATTERNS [IGNORE_PATTERNS ...]] + [--use-tqdm-on-load | --no-use-tqdm-on-load] + [--pt-load-map-location PT_LOAD_MAP_LOCATION] + [--attention-backend ATTENTION_BACKEND] + [--reasoning-parser REASONING_PARSER] + [--reasoning-parser-plugin REASONING_PARSER_PLUGIN] + [--distributed-executor-backend ['external_launcher', 'mp', 'ray', 'uni']] + [--pipeline-parallel-size PIPELINE_PARALLEL_SIZE] + [--master-addr MASTER_ADDR] [--master-port MASTER_PORT] + [--nnodes NNODES] [--node-rank NODE_RANK] + [--tensor-parallel-size TENSOR_PARALLEL_SIZE] + [--decode-context-parallel-size DECODE_CONTEXT_PARALLEL_SIZE] + [--dcp-kv-cache-interleave-size DCP_KV_CACHE_INTERLEAVE_SIZE] + [--cp-kv-cache-interleave-size CP_KV_CACHE_INTERLEAVE_SIZE] + [--prefill-context-parallel-size PREFILL_CONTEXT_PARALLEL_SIZE] + [--data-parallel-size DATA_PARALLEL_SIZE] + [--data-parallel-rank DATA_PARALLEL_RANK] + [--data-parallel-start-rank DATA_PARALLEL_START_RANK] + [--data-parallel-size-local DATA_PARALLEL_SIZE_LOCAL] + [--data-parallel-address DATA_PARALLEL_ADDRESS] + [--data-parallel-rpc-port DATA_PARALLEL_RPC_PORT] + [--data-parallel-backend DATA_PARALLEL_BACKEND] + [--data-parallel-hybrid-lb | --no-data-parallel-hybrid-lb | -dph] + [--data-parallel-external-lb | --no-data-parallel-external-lb | -dpe] + [--enable-expert-parallel | --no-enable-expert-parallel] + [--all2all-backend {allgather_reducescatter,deepep_high_throughput,deepep_low_latency,flashinfer_all2allv,naive,pplx,None}] + [--enable-dbo | --no-enable-dbo] + [--dbo-decode-token-threshold DBO_DECODE_TOKEN_THRESHOLD] + [--dbo-prefill-token-threshold DBO_PREFILL_TOKEN_THRESHOLD] + [--disable-nccl-for-dp-synchronization | --no-disable-nccl-for-dp-synchronization] + [--enable-eplb | --no-enable-eplb] + [--eplb-config EPLB_CONFIG] + [--expert-placement-strategy {linear,round_robin}] + [--max-parallel-loading-workers MAX_PARALLEL_LOADING_WORKERS] + [--ray-workers-use-nsight | --no-ray-workers-use-nsight] + [--disable-custom-all-reduce | --no-disable-custom-all-reduce] + [--worker-cls WORKER_CLS] + [--worker-extension-cls WORKER_EXTENSION_CLS] + [--block-size {1,8,16,32,64,128,256}] + [--gpu-memory-utilization GPU_MEMORY_UTILIZATION] + [--kv-cache-memory-bytes KV_CACHE_MEMORY_BYTES] + [--swap-space SWAP_SPACE] + [--kv-cache-dtype {auto,bfloat16,fp8,fp8_ds_mla,fp8_e4m3,fp8_e5m2,fp8_inc}] + [--num-gpu-blocks-override NUM_GPU_BLOCKS_OVERRIDE] + [--enable-prefix-caching | --no-enable-prefix-caching] + [--prefix-caching-hash-algo {sha256,sha256_cbor,xxhash,xxhash_cbor}] + [--cpu-offload-gb CPU_OFFLOAD_GB] + [--calculate-kv-scales | --no-calculate-kv-scales] + [--kv-sharing-fast-prefill | --no-kv-sharing-fast-prefill] + [--mamba-cache-dtype {auto,float16,float32}] + [--mamba-ssm-cache-dtype {auto,float16,float32}] + [--mamba-block-size MAMBA_BLOCK_SIZE] + [--kv-offloading-size KV_OFFLOADING_SIZE] + [--kv-offloading-backend {lmcache,native,None}] + [--limit-mm-per-prompt LIMIT_MM_PER_PROMPT] + [--enable-mm-embeds | --no-enable-mm-embeds] + [--media-io-kwargs MEDIA_IO_KWARGS] + [--mm-processor-kwargs MM_PROCESSOR_KWARGS] + [--mm-processor-cache-gb MM_PROCESSOR_CACHE_GB] + [--mm-processor-cache-type {lru,shm}] + [--mm-shm-cache-max-object-size-mb MM_SHM_CACHE_MAX_OBJECT_SIZE_MB] + [--mm-encoder-tp-mode {data,weights}] + [--mm-encoder-attn-backend MM_ENCODER_ATTN_BACKEND] + [--interleave-mm-strings | --no-interleave-mm-strings] + [--skip-mm-profiling | --no-skip-mm-profiling] + [--video-pruning-rate VIDEO_PRUNING_RATE] + [--enable-lora | --no-enable-lora] + [--max-loras MAX_LORAS] + [--max-lora-rank {1,8,16,32,64,128,256,320,512}] + [--lora-dtype {auto,bfloat16,float16}] + [--max-cpu-loras MAX_CPU_LORAS] + [--fully-sharded-loras | --no-fully-sharded-loras] + [--default-mm-loras DEFAULT_MM_LORAS] + [--show-hidden-metrics-for-version SHOW_HIDDEN_METRICS_FOR_VERSION] + [--otlp-traces-endpoint OTLP_TRACES_ENDPOINT] + [--collect-detailed-traces {all,model,worker,None} [{all,model,worker,None} ...]] + [--kv-cache-metrics | --no-kv-cache-metrics] + [--kv-cache-metrics-sample KV_CACHE_METRICS_SAMPLE] + [--cudagraph-metrics | --no-cudagraph-metrics] + [--enable-layerwise-nvtx-tracing | --no-enable-layerwise-nvtx-tracing] + [--max-num-batched-tokens MAX_NUM_BATCHED_TOKENS] + [--max-num-seqs MAX_NUM_SEQS] + [--max-num-partial-prefills MAX_NUM_PARTIAL_PREFILLS] + [--max-long-partial-prefills MAX_LONG_PARTIAL_PREFILLS] + [--long-prefill-token-threshold LONG_PREFILL_TOKEN_THRESHOLD] + [--scheduling-policy {fcfs,priority}] + [--enable-chunked-prefill | --no-enable-chunked-prefill] + [--disable-chunked-mm-input | --no-disable-chunked-mm-input] + [--scheduler-cls SCHEDULER_CLS] + [--disable-hybrid-kv-cache-manager | --no-disable-hybrid-kv-cache-manager] + [--async-scheduling | --no-async-scheduling] + [--stream-interval STREAM_INTERVAL] + [--cudagraph-capture-sizes CUDAGRAPH_CAPTURE_SIZES [CUDAGRAPH_CAPTURE_SIZES ...]] + [--max-cudagraph-capture-size MAX_CUDAGRAPH_CAPTURE_SIZE] + [--speculative-config SPECULATIVE_CONFIG] + [--kv-transfer-config KV_TRANSFER_CONFIG] + [--kv-events-config KV_EVENTS_CONFIG] + [--ec-transfer-config EC_TRANSFER_CONFIG] + [--compilation-config COMPILATION_CONFIG] + [--attention-config ATTENTION_CONFIG] + [--additional-config ADDITIONAL_CONFIG] + [--structured-outputs-config STRUCTURED_OUTPUTS_CONFIG] + [--profiler-config PROFILER_CONFIG] + [--optimization-level OPTIMIZATION_LEVEL] + [--disable-log-stats] [--aggregate-engine-logging] + [--enable-log-requests | --no-enable-log-requests] + [--disable-log-requests | --no-disable-log-requests] + [model_tag] +api_server.py: error: unrecognized arguments: --guided-decoding-backend +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:06<00:19, 6.52s/it] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:08<00:07, 3.70s/it] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:15<00:05, 5.28s/it] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:21<00:00, 5.57s/it] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:21<00:00, 5.35s/it] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 14.83it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:03, 15.62it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 16.45it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 16.87it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 18.23it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 19.00it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.74it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.74it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.38it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.90it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:01, 20.09it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 21.04it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 21.92it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 22.58it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:01<00:00, 23.20it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 23.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 23.47it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 24.19it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 20.91it/s] +[0;36m(EngineCore_DP0 pid=3603174)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:04, 7.51it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:01, 16.43it/s]
Capturing CUDA graphs (decode, FULL): 20%|██ | 7/35 [00:00<00:01, 19.40it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:01, 20.92it/s]
Capturing CUDA graphs (decode, FULL): 37%|███▋ | 13/35 [00:00<00:01, 21.93it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:00<00:00, 22.62it/s]
Capturing CUDA graphs (decode, FULL): 54%|█████▍ | 19/35 [00:00<00:00, 23.03it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:01<00:00, 23.33it/s]
Capturing CUDA graphs (decode, FULL): 71%|███████▏ | 25/35 [00:01<00:00, 23.31it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:01<00:00, 23.67it/s]
Capturing CUDA graphs (decode, FULL): 89%|████████▊ | 31/35 [00:01<00:00, 24.00it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:01<00:00, 24.21it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 22.44it/s] +[0;36m(APIServer pid=3602630)[0;0m INFO: Started server process [3602630] +[0;36m(APIServer pid=3602630)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=3602630)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-26 12:38:05,935 - INFO - Loaded dataset: math-hard +2026-01-26 12:38:05,935 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 12:38:06,001 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-26 12:38:06,003 - INFO - Running method: vanilla +2026-01-26 12:38:06,004 - INFO - Running profiles 0 to 9 (10 remaining) +2026-01-26 12:38:06,004 - INFO - Using BATCH processing (local vLLM user) for vanilla +2026-01-26 12:38:06,006 - INFO - Using local vLLM user simulator: http://localhost:8004/v1 +2026-01-26 12:38:11,230 - INFO - Batch: 10 profiles, 20 sessions remaining +2026-01-26 12:38:11,240 - INFO - Session round 1/2: 10 total, 6876 sessions/hr +2026-01-26 12:38:11,248 - INFO - Session round 2/2: 20 total, 13730 sessions/hr +2026-01-26 12:38:11,996 - INFO - GPU memory freed after vanilla: 0.0GB allocated +2026-01-26 12:38:12,000 - INFO - Report saved to ../results/test_local_user_20260126_123755/20260126_123805/report.md diff --git a/collaborativeagents/slurm/fullscale/test_run.sh b/collaborativeagents/slurm/fullscale/test_run.sh new file mode 100644 index 0000000..de6a0e1 --- /dev/null +++ b/collaborativeagents/slurm/fullscale/test_run.sh @@ -0,0 +1,70 @@ +#!/bin/bash +#SBATCH --job-name=test_fullscale +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=1:00:00 +#SBATCH --output=test_fullscale_%j.out +#SBATCH --error=test_fullscale_%j.err + +# Test run: 2 profiles × 2 methods × 2 sessions = 8 sessions +# Should complete in ~10-15 minutes + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +# Load OpenAI API key +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# GPU 0: vLLM 8B agent, GPU 1: adapter models +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +# Wait for server +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM 8B agent server ready." + +cd collaborativeagents/scripts + +# Test run: vanilla + rag_vector (light + heavy methods) +python run_experiments.py \ + --methods vanilla,rag_vector \ + --datasets math-hard \ + --n-profiles 2 \ + --n-sessions 2 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5-mini \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 2 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/fullscale_test_$(date +%Y%m%d_%H%M%S) + +echo "Test run complete!" + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/generate_profiles.sh b/collaborativeagents/slurm/generate_profiles.sh new file mode 100644 index 0000000..936a1e6 --- /dev/null +++ b/collaborativeagents/slurm/generate_profiles.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#SBATCH --job-name=gen_profiles +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=cpu +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=16G +#SBATCH --time=00:30:00 +#SBATCH --output=logs/gen_profiles_%j.out +#SBATCH --error=logs/gen_profiles_%j.err + +# Generate 100 user profiles from schema (no LLM required) +# This is fast and doesn't need GPU + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs directory +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/data/complex_profiles_v2 + +echo "Starting profile generation at $(date)" +echo "Job ID: $SLURM_JOB_ID" + +# Use the eval environment (has required packages) +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Generate profiles from schema (no LLM needed) +python collaborativeagents/scripts/generate_profiles_v2.py \ + --num_profiles 100 \ + --from_schema collaborativeagents/data/preference_schema_v2_sample.json \ + --output collaborativeagents/data/complex_profiles_v2/profiles_100.jsonl \ + --seed 42 + +echo "Profile generation completed at $(date)" +echo "Output: collaborativeagents/data/complex_profiles_v2/profiles_100.jsonl" diff --git a/collaborativeagents/slurm/logs/all_memory_14360420.err b/collaborativeagents/slurm/logs/all_memory_14360420.err new file mode 100644 index 0000000..93582b3 --- /dev/null +++ b/collaborativeagents/slurm/logs/all_memory_14360420.err @@ -0,0 +1,127 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:56:00,092 - INFO - Loaded dataset: mmlu +2025-12-27 01:56:00,092 - INFO - Loaded dataset: aime +2025-12-27 01:56:00,092 - INFO - Loaded dataset: math-hard +2025-12-27 01:56:00,092 - INFO - Loaded dataset: humaneval +2025-12-27 01:56:00,108 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:56:00,108 - INFO - Running method: all_memory +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.35s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 6.00s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.61s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:14, 3.74s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.53s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.06s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.22s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.39s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.87s/it] +2025-12-27 01:57:24,440 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:05<00:41, 5.24s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:11<00:41, 5.96s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:22<00:49, 8.20s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:33<00:45, 9.10s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:51<00:49, 12.45s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:01<00:35, 11.68s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:08<00:20, 10.17s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:15<00:08, 9.00s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:17<00:00, 6.98s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:17<00:00, 8.64s/it] +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-27 02:02:45,722 - WARNING - User agent failed to respond at turn 4 +2025-12-27 02:09:29,423 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:38:42,128 - WARNING - User agent failed to respond at turn 2 +2025-12-27 02:51:47,964 - INFO - Profile 2/30 +2025-12-27 03:33:09,939 - INFO - Profile 3/30 +2025-12-27 03:37:34,526 - WARNING - User agent failed to respond at turn 4 +2025-12-27 04:17:59,068 - WARNING - User agent failed to respond at turn 6 +2025-12-27 04:27:33,697 - INFO - Profile 4/30 +2025-12-27 04:54:07,594 - WARNING - User agent failed to respond at turn 4 +2025-12-27 05:15:55,995 - WARNING - User agent failed to respond at turn 3 +2025-12-27 05:31:32,714 - INFO - Profile 5/30 +2025-12-27 06:00:08,138 - INFO - Profile 6/30 +2025-12-27 06:26:50,737 - WARNING - User agent failed to respond at turn 4 +2025-12-27 06:48:12,610 - INFO - Profile 7/30 +2025-12-27 06:50:45,060 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:09:39,293 - WARNING - User agent failed to respond at turn 6 +2025-12-27 07:21:31,085 - WARNING - User agent failed to respond at turn 4 +2025-12-27 07:31:39,376 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:50:16,030 - INFO - Profile 8/30 +2025-12-27 08:08:45,945 - WARNING - User agent failed to respond at turn 5 +2025-12-27 08:12:05,648 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:35:55,463 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:44:51,942 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:54:22,336 - INFO - Profile 9/30 +2025-12-27 09:27:31,666 - WARNING - User agent failed to respond at turn 7 +2025-12-27 09:38:06,322 - INFO - Profile 10/30 +2025-12-27 10:42:39,399 - INFO - Profile 11/30 +2025-12-27 11:30:24,395 - WARNING - User agent failed to respond at turn 6 +2025-12-27 11:47:24,803 - WARNING - User agent failed to respond at turn 4 +2025-12-27 11:47:24,805 - INFO - Profile 12/30 +2025-12-27 11:54:27,915 - WARNING - User agent failed to respond at turn 8 +2025-12-27 12:45:25,244 - INFO - Profile 13/30 +2025-12-27 13:25:06,083 - INFO - Profile 14/30 +2025-12-27 13:59:04,923 - WARNING - User agent failed to respond at turn 4 +2025-12-27 14:06:28,017 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:08:43,353 - INFO - Profile 15/30 +2025-12-27 14:19:46,215 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:56:28,404 - WARNING - User agent failed to respond at turn 5 +2025-12-27 15:02:31,026 - INFO - Profile 16/30 +2025-12-27 15:41:30,611 - INFO - Profile 17/30 +2025-12-27 16:22:32,360 - WARNING - User agent failed to respond at turn 3 +2025-12-27 16:25:57,033 - WARNING - User agent failed to respond at turn 3 +2025-12-27 16:28:31,989 - WARNING - User agent failed to respond at turn 0 +2025-12-27 16:35:53,597 - WARNING - User agent failed to respond at turn 3 +2025-12-27 16:35:53,598 - INFO - Profile 18/30 +2025-12-27 16:53:53,194 - WARNING - User agent failed to respond at turn 2 +2025-12-27 16:59:54,367 - WARNING - User agent failed to respond at turn 2 +2025-12-27 17:26:40,246 - INFO - Profile 19/30 +2025-12-27 17:29:49,305 - WARNING - User agent failed to respond at turn 3 +2025-12-27 17:34:00,096 - WARNING - User agent failed to respond at turn 6 +2025-12-27 18:29:51,532 - WARNING - User agent failed to respond at turn 3 +2025-12-27 18:34:52,391 - INFO - Profile 20/30 +2025-12-27 19:24:35,799 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:40:55,850 - INFO - Profile 21/30 +2025-12-27 19:45:01,108 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:56:02,480 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:59:59,405 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:12:20,145 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:16:09,511 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:24:18,687 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:41:10,014 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:45:51,169 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:54:31,036 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:59:18,893 - WARNING - User agent failed to respond at turn 3 +2025-12-27 21:09:55,958 - WARNING - User agent failed to respond at turn 3 +2025-12-27 21:09:55,960 - INFO - Profile 22/30 +2025-12-27 21:39:02,331 - WARNING - User agent failed to respond at turn 6 +2025-12-27 22:07:27,301 - INFO - Profile 23/30 +2025-12-27 23:18:34,663 - INFO - Profile 24/30 +2025-12-27 23:36:19,489 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:40:12,672 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:53:47,032 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:10:26,896 - INFO - Profile 25/30 +2025-12-28 00:17:50,523 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:34:10,018 - WARNING - User agent failed to respond at turn 5 +2025-12-28 00:56:33,834 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:10:01,462 - INFO - Profile 26/30 +2025-12-28 01:26:26,996 - WARNING - User agent failed to respond at turn 5 +2025-12-28 01:32:18,640 - WARNING - User agent failed to respond at turn 5 +2025-12-28 01:43:28,926 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:54:35,490 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:58:33,822 - WARNING - User agent failed to respond at turn 5 +2025-12-28 02:15:30,859 - INFO - Profile 27/30 +2025-12-28 02:29:36,631 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:33:19,067 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:46:48,847 - WARNING - User agent failed to respond at turn 6 +2025-12-28 02:55:44,372 - WARNING - User agent failed to respond at turn 3 +2025-12-28 03:25:07,911 - INFO - Profile 28/30 +2025-12-28 04:17:59,003 - WARNING - User agent failed to respond at turn 7 +2025-12-28 04:28:02,080 - INFO - Profile 29/30 +2025-12-28 04:33:42,783 - WARNING - User agent failed to respond at turn 2 +2025-12-28 04:39:07,085 - WARNING - User agent failed to respond at turn 4 +2025-12-28 04:42:42,813 - WARNING - User agent failed to respond at turn 4 +2025-12-28 05:10:55,776 - WARNING - User agent failed to respond at turn 3 +2025-12-28 05:30:48,797 - INFO - Profile 30/30 +2025-12-28 06:06:54,294 - WARNING - User agent failed to respond at turn 3 +2025-12-28 06:19:26,579 - INFO - Report saved to ../results/all_memory_20251227_015537/20251227_015600/report.md diff --git a/collaborativeagents/slurm/logs/contextual_14360423.err b/collaborativeagents/slurm/logs/contextual_14360423.err new file mode 100644 index 0000000..ae2ad9b --- /dev/null +++ b/collaborativeagents/slurm/logs/contextual_14360423.err @@ -0,0 +1,139 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 02:02:06,308 - INFO - Loaded dataset: mmlu +2025-12-27 02:02:06,308 - INFO - Loaded dataset: aime +2025-12-27 02:02:06,308 - INFO - Loaded dataset: math-hard +2025-12-27 02:02:06,308 - INFO - Loaded dataset: humaneval +2025-12-27 02:02:06,329 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 02:02:06,329 - INFO - Running method: contextual +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:21, 7.14s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:12, 6.50s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.40s/it] +2025-12-27 02:02:32,180 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:04<00:33, 4.15s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:08<00:31, 4.52s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:13<00:28, 4.68s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:18<00:24, 4.81s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:23<00:19, 4.88s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:28<00:14, 4.91s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [00:33<00:09, 4.97s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [00:37<00:04, 4.44s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 3.63s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 4.34s/it] +2025-12-27 02:19:52,764 - WARNING - User agent failed to respond at turn 8 +2025-12-27 02:34:35,957 - WARNING - User agent failed to respond at turn 9 +2025-12-27 02:42:19,862 - WARNING - User agent failed to respond at turn 6 +2025-12-27 02:50:50,883 - WARNING - User agent failed to respond at turn 4 +2025-12-27 03:04:43,668 - WARNING - User agent failed to respond at turn 8 +2025-12-27 03:18:17,893 - WARNING - User agent failed to respond at turn 8 +2025-12-27 03:31:55,124 - WARNING - User agent failed to respond at turn 9 +2025-12-27 03:47:00,955 - WARNING - User agent failed to respond at turn 9 +2025-12-27 04:14:22,548 - WARNING - User agent failed to respond at turn 9 +2025-12-27 04:37:23,679 - WARNING - User agent failed to respond at turn 8 +2025-12-27 04:40:56,780 - WARNING - User agent failed to respond at turn 2 +2025-12-27 05:05:21,340 - WARNING - User agent failed to respond at turn 8 +2025-12-27 05:19:26,915 - WARNING - User agent failed to respond at turn 8 +2025-12-27 05:34:10,737 - WARNING - User agent failed to respond at turn 10 +2025-12-27 05:46:47,804 - WARNING - User agent failed to respond at turn 5 +'(ReadTimeoutError("HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)"), '(Request ID: 65a54fd4-dc47-449f-99f1-8b72cd64f258)')' thrown while requesting HEAD https://huggingface.co/datasets/lighteval/MATH-Hard/resolve/cf0716b8bafa192bcb6b455b0679538787dc43f0/.huggingface.yaml +2025-12-27 05:47:08,075 - WARNING - '(ReadTimeoutError("HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)"), '(Request ID: 65a54fd4-dc47-449f-99f1-8b72cd64f258)')' thrown while requesting HEAD https://huggingface.co/datasets/lighteval/MATH-Hard/resolve/cf0716b8bafa192bcb6b455b0679538787dc43f0/.huggingface.yaml +Retrying in 1s [Retry 1/5]. +2025-12-27 05:47:08,075 - WARNING - Retrying in 1s [Retry 1/5]. +2025-12-27 05:47:31,159 - INFO - Profile 2/30 +2025-12-27 06:06:02,584 - WARNING - User agent failed to respond at turn 8 +2025-12-27 06:28:56,672 - WARNING - User agent failed to respond at turn 9 +2025-12-27 06:33:43,493 - WARNING - User agent failed to respond at turn 3 +2025-12-27 06:55:51,361 - WARNING - User agent failed to respond at turn 8 +2025-12-27 07:04:21,768 - WARNING - User agent failed to respond at turn 9 +2025-12-27 07:36:32,573 - WARNING - User agent failed to respond at turn 6 +2025-12-27 08:03:58,605 - WARNING - User agent failed to respond at turn 10 +2025-12-27 08:31:36,913 - WARNING - User agent failed to respond at turn 8 +2025-12-27 08:45:06,689 - WARNING - User agent failed to respond at turn 10 +2025-12-27 08:57:40,880 - WARNING - User agent failed to respond at turn 8 +2025-12-27 09:09:30,300 - WARNING - User agent failed to respond at turn 10 +2025-12-27 09:09:30,302 - INFO - Profile 3/30 +2025-12-27 09:23:09,110 - WARNING - User agent failed to respond at turn 9 +2025-12-27 09:38:33,087 - WARNING - User agent failed to respond at turn 10 +2025-12-27 09:55:31,104 - WARNING - User agent failed to respond at turn 10 +2025-12-27 10:08:01,546 - WARNING - User agent failed to respond at turn 10 +2025-12-27 10:11:50,915 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:20:33,908 - WARNING - User agent failed to respond at turn 8 +2025-12-27 10:36:06,815 - WARNING - User agent failed to respond at turn 13 +2025-12-27 11:04:44,557 - WARNING - User agent failed to respond at turn 9 +2025-12-27 11:19:08,586 - WARNING - User agent failed to respond at turn 10 +2025-12-27 11:32:30,613 - WARNING - User agent failed to respond at turn 8 +2025-12-27 11:45:28,346 - WARNING - User agent failed to respond at turn 8 +2025-12-27 11:59:10,443 - WARNING - User agent failed to respond at turn 8 +2025-12-27 12:02:44,417 - WARNING - User agent failed to respond at turn 2 +2025-12-27 12:24:05,983 - WARNING - User agent failed to respond at turn 9 +2025-12-27 12:36:47,157 - WARNING - User agent failed to respond at turn 8 +2025-12-27 12:56:50,763 - WARNING - User agent failed to respond at turn 8 +2025-12-27 13:14:23,987 - INFO - Profile 4/30 +2025-12-27 13:41:52,436 - WARNING - User agent failed to respond at turn 8 +2025-12-27 14:02:43,583 - WARNING - User agent failed to respond at turn 8 +2025-12-27 14:06:32,824 - WARNING - User agent failed to respond at turn 2 +2025-12-27 14:23:22,832 - WARNING - User agent failed to respond at turn 14 +2025-12-27 14:37:30,153 - WARNING - User agent failed to respond at turn 8 +2025-12-27 14:51:10,701 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:04:43,741 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:19:48,665 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:33:01,051 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:49:22,254 - WARNING - User agent failed to respond at turn 8 +2025-12-27 16:03:56,686 - WARNING - User agent failed to respond at turn 8 +2025-12-27 16:20:53,945 - WARNING - User agent failed to respond at turn 7 +2025-12-27 16:49:33,698 - WARNING - User agent failed to respond at turn 8 +2025-12-27 17:04:50,896 - WARNING - User agent failed to respond at turn 9 +2025-12-27 17:31:00,796 - WARNING - User agent failed to respond at turn 8 +2025-12-27 17:31:00,799 - INFO - Profile 5/30 +2025-12-27 18:00:49,422 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:15:01,789 - WARNING - User agent failed to respond at turn 10 +2025-12-27 18:27:47,576 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:40:13,645 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:53:15,915 - WARNING - User agent failed to respond at turn 8 +2025-12-27 19:19:08,201 - WARNING - User agent failed to respond at turn 8 +2025-12-27 19:37:35,846 - WARNING - User agent failed to respond at turn 8 +2025-12-27 19:51:06,794 - WARNING - User agent failed to respond at turn 10 +2025-12-27 20:05:06,821 - WARNING - User agent failed to respond at turn 10 +2025-12-27 20:17:39,488 - WARNING - User agent failed to respond at turn 8 +2025-12-27 20:31:13,429 - WARNING - User agent failed to respond at turn 7 +2025-12-27 20:53:38,048 - WARNING - User agent failed to respond at turn 9 +2025-12-27 21:06:30,110 - WARNING - User agent failed to respond at turn 9 +2025-12-27 21:18:58,273 - WARNING - User agent failed to respond at turn 8 +2025-12-27 21:31:52,874 - WARNING - User agent failed to respond at turn 8 +2025-12-27 21:45:02,984 - WARNING - User agent failed to respond at turn 8 +2025-12-27 21:45:02,987 - INFO - Profile 6/30 +2025-12-27 21:59:20,567 - WARNING - User agent failed to respond at turn 12 +2025-12-27 22:12:05,436 - WARNING - User agent failed to respond at turn 9 +2025-12-27 22:24:26,612 - WARNING - User agent failed to respond at turn 8 +2025-12-27 22:37:34,213 - WARNING - User agent failed to respond at turn 8 +2025-12-27 22:41:11,469 - WARNING - User agent failed to respond at turn 2 +2025-12-27 22:54:18,480 - WARNING - User agent failed to respond at turn 8 +2025-12-27 22:57:33,805 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:09:44,313 - WARNING - User agent failed to respond at turn 9 +2025-12-27 23:22:24,512 - WARNING - User agent failed to respond at turn 8 +2025-12-27 23:35:10,690 - WARNING - User agent failed to respond at turn 8 +2025-12-27 23:51:25,271 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:04:32,550 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:17:05,022 - WARNING - User agent failed to respond at turn 7 +2025-12-28 00:29:45,695 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:42:31,155 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:55:35,641 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:09:42,819 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:22:54,235 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:35:57,818 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:35:57,820 - INFO - Profile 7/30 +2025-12-28 02:29:13,033 - WARNING - User agent failed to respond at turn 8 +2025-12-28 02:34:18,064 - WARNING - User agent failed to respond at turn 2 +2025-12-28 02:47:50,033 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:01:20,372 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:17:29,349 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:31:22,192 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:44:57,553 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:59:52,603 - WARNING - User agent failed to respond at turn 8 +[2025-12-28T04:23:53.546] error: *** JOB 14360423 ON gpua091 CANCELLED AT 2025-12-28T04:23:53 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/full_exp_14357783.err b/collaborativeagents/slurm/logs/full_exp_14357783.err new file mode 100644 index 0000000..a0c7ce5 --- /dev/null +++ b/collaborativeagents/slurm/logs/full_exp_14357783.err @@ -0,0 +1,69 @@ +2025-12-26 02:38:46,860 - INFO - Loaded dataset: gpqa +2025-12-26 02:38:46,861 - INFO - Loaded dataset: aime +2025-12-26 02:38:46,861 - INFO - Loaded dataset: math-hard +2025-12-26 02:38:46,861 - INFO - Loaded dataset: humaneval +2025-12-26 02:38:46,872 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-26 02:38:46,872 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:09<00:27, 9.02s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:11, 5.81s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.42s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.37s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:16, 4.19s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:10, 3.44s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.63s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.44s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.65s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.07s/it] +2025-12-26 02:39:34,147 - INFO - Profile 1/30 +
Generating train split: 0%| | 0/90 [00:00<?, ? examples/s]
Generating train split: 100%|██████████| 90/90 [00:00<00:00, 1116.85 examples/s] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:05<00:41, 5.22s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:13<00:49, 7.11s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:23<00:48, 8.17s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:33<00:44, 8.98s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:45<00:40, 10.19s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:56<00:31, 10.44s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:04<00:19, 9.67s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:08<00:07, 7.94s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:10<00:00, 5.95s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:10<00:00, 7.83s/it] +2025-12-26 02:40:51,255 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,261 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,266 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,271 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,276 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,281 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,286 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,291 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,296 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,301 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,306 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,310 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,315 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,320 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,326 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,330 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,335 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,341 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,345 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,350 - WARNING - User agent failed to respond at turn 0 +
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 2304 examples [00:00, 29493.27 examples/s] +
Generating test split: 0 examples [00:00, ? examples/s]
Generating test split: 1324 examples [00:00, 28145.98 examples/s] +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 623, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 608, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 414, in run_all + results = self.run_method(method) + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 367, in run_method + samples = dataset.get_testset() + ^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/datasets_extended.py", line 71, in get_testset + self._test_data = self._load_data("test")[:self.eval_size] + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/datasets_extended.py", line 153, in _load_data + solution=item["answer"], + ~~~~^^^^^^^^^^ +KeyError: 'answer' diff --git a/collaborativeagents/slurm/logs/full_exp_14358390.err b/collaborativeagents/slurm/logs/full_exp_14358390.err new file mode 100644 index 0000000..65669a6 --- /dev/null +++ b/collaborativeagents/slurm/logs/full_exp_14358390.err @@ -0,0 +1,97 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-26 11:03:58,116 - INFO - Loaded dataset: mmlu +2025-12-26 11:03:58,116 - INFO - Loaded dataset: aime +2025-12-26 11:03:58,116 - INFO - Loaded dataset: math-hard +2025-12-26 11:03:58,116 - INFO - Loaded dataset: humaneval +2025-12-26 11:03:58,213 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-26 11:03:58,214 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:16, 5.63s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.11s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 5.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.87s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.49s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:23, 5.84s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.19s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:19<00:04, 4.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:21<00:00, 3.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:21<00:00, 4.25s/it] +2025-12-26 11:04:49,945 - INFO - Profile 1/30 +
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 1179.52 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3170.95 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1567.85 examples/s] +
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 21317.94 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3131.14 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1635.46 examples/s] +
Generating test split: 0%| | 0/102 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 102/102 [00:00<00:00, 22671.91 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3275.40 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1562.01 examples/s] +
Generating test split: 0%| | 0/126 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 126/126 [00:00<00:00, 22251.89 examples/s] +
Generating validation split: 0%| | 0/14 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 14/14 [00:00<00:00, 3442.39 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1582.64 examples/s] +
Generating test split: 0%| | 0/151 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 151/151 [00:00<00:00, 33935.59 examples/s] +
Generating validation split: 0%| | 0/17 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 17/17 [00:00<00:00, 5068.47 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1145.67 examples/s] +
Generating test split: 0%| | 0/112 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 112/112 [00:00<00:00, 28756.25 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 1310.68 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1548.86 examples/s] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:05<00:44, 5.50s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [04:02<16:31, 141.68s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [04:07<07:54, 79.14s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [04:13<04:11, 50.20s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [04:18<02:16, 34.13s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [04:24<01:13, 24.38s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [05:50<01:29, 44.64s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [05:53<00:31, 31.48s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [05:55<00:00, 22.11s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [05:55<00:00, 39.48s/it] +2025-12-26 11:22:36,555 - WARNING - User agent failed to respond at turn 3 +2025-12-26 11:38:11,820 - WARNING - User agent failed to respond at turn 4 +2025-12-26 11:49:23,100 - WARNING - User agent failed to respond at turn 2 +2025-12-26 12:02:46,057 - INFO - Profile 2/30 +2025-12-26 12:15:14,260 - WARNING - User agent failed to respond at turn 5 +2025-12-26 12:43:01,887 - INFO - Profile 3/30 +2025-12-26 13:16:25,157 - WARNING - User agent failed to respond at turn 3 +2025-12-26 13:19:41,611 - WARNING - User agent failed to respond at turn 4 +2025-12-26 13:31:32,934 - INFO - Profile 4/30 +2025-12-26 13:44:18,881 - WARNING - User agent failed to respond at turn 2 +2025-12-26 13:50:24,735 - WARNING - User agent failed to respond at turn 3 +2025-12-26 14:22:35,201 - WARNING - User agent failed to respond at turn 4 +2025-12-26 14:28:33,134 - INFO - Profile 5/30 +2025-12-26 15:08:47,500 - INFO - Profile 6/30 +2025-12-26 15:11:23,025 - WARNING - User agent failed to respond at turn 4 +2025-12-26 15:16:01,352 - WARNING - User agent failed to respond at turn 3 +2025-12-26 15:42:13,161 - WARNING - User agent failed to respond at turn 4 +2025-12-26 15:46:22,086 - WARNING - User agent failed to respond at turn 6 +2025-12-26 15:56:17,527 - WARNING - User agent failed to respond at turn 4 +2025-12-26 16:07:50,432 - WARNING - User agent failed to respond at turn 6 +2025-12-26 16:07:50,434 - INFO - Profile 7/30 +2025-12-26 16:17:55,588 - WARNING - User agent failed to respond at turn 2 +2025-12-26 16:33:05,425 - WARNING - User agent failed to respond at turn 3 +2025-12-26 16:56:24,660 - WARNING - User agent failed to respond at turn 3 +2025-12-26 17:08:04,806 - INFO - Profile 8/30 +2025-12-26 18:12:23,818 - WARNING - User agent failed to respond at turn 3 +2025-12-26 18:15:59,440 - INFO - Profile 9/30 +2025-12-26 18:25:15,857 - WARNING - User agent failed to respond at turn 3 +2025-12-26 18:27:49,419 - WARNING - User agent failed to respond at turn 3 +2025-12-26 18:57:08,009 - INFO - Profile 10/30 +2025-12-26 19:36:53,452 - WARNING - User agent failed to respond at turn 2 +2025-12-26 19:54:08,446 - INFO - Profile 11/30 +2025-12-26 20:09:55,735 - WARNING - User agent failed to respond at turn 3 +2025-12-26 20:40:31,861 - INFO - Profile 12/30 +2025-12-26 21:14:56,537 - WARNING - User agent failed to respond at turn 3 +2025-12-26 21:28:00,763 - WARNING - User agent failed to respond at turn 3 +2025-12-26 21:28:00,764 - INFO - Profile 13/30 +2025-12-26 22:12:17,605 - INFO - Profile 14/30 +2025-12-26 22:35:24,416 - WARNING - User agent failed to respond at turn 4 +2025-12-26 22:59:55,984 - INFO - Profile 15/30 +2025-12-26 23:10:51,206 - WARNING - User agent failed to respond at turn 2 +2025-12-26 23:23:48,320 - WARNING - User agent failed to respond at turn 3 +2025-12-26 23:41:06,303 - INFO - Profile 16/30 +2025-12-27 00:26:43,745 - INFO - Profile 17/30 +2025-12-27 00:51:28,080 - WARNING - User agent failed to respond at turn 5 +2025-12-27 01:04:15,959 - WARNING - User agent failed to respond at turn 2 +2025-12-27 01:11:14,362 - WARNING - User agent failed to respond at turn 3 +2025-12-27 01:16:42,254 - WARNING - User agent failed to respond at turn 3 +2025-12-27 01:22:35,316 - INFO - Profile 18/30 +[2025-12-27T01:24:47.209] error: *** JOB 14358390 ON gpua035 CANCELLED AT 2025-12-27T01:24:47 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/rag_14360421.err b/collaborativeagents/slurm/logs/rag_14360421.err new file mode 100644 index 0000000..de70ef7 --- /dev/null +++ b/collaborativeagents/slurm/logs/rag_14360421.err @@ -0,0 +1,9442 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:58:02,188 - INFO - Loaded dataset: mmlu +2025-12-27 01:58:02,188 - INFO - Loaded dataset: aime +2025-12-27 01:58:02,188 - INFO - Loaded dataset: math-hard +2025-12-27 01:58:02,188 - INFO - Loaded dataset: humaneval +2025-12-27 01:58:02,201 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:58:02,201 - INFO - Running method: rag +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:21, 7.24s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.15s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.70s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.99s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.17s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.07s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.14s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.82s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.24s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.58s/it] +2025-12-27 01:58:53,808 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:04<00:38, 4.87s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:12<00:45, 6.46s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:23<00:50, 8.41s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:33<00:46, 9.26s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:47<00:43, 10.88s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:57<00:31, 10.56s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:04<00:18, 9.48s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:12<00:08, 8.89s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:15<00:00, 7.14s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:15<00:00, 8.40s/it] +[2025-12-27T02:03:19.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:06:19.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:07:51.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:12.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:15.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:25.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:43.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:47.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:49.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:11.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:15.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:17.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:48.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:50.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:52.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:33.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:35.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:37.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:39.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:43.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:45.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:12.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:15.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:25.126] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:43.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:47.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:49.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:14.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:16.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:18.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:42.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:44.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:46.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:13.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:17.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:19.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:40.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:44.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:46.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:14.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:16.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:26.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:47.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:49.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:51.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:16.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:18.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:20.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:41.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:43.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:45.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:14.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:16.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:26.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:44.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:48.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:50.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:15.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:17.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:19.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:42.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:44.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:46.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:14.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:16.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:26.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:46.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:48.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:50.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:15.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:17.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:19.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:42.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:46.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:14.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:17.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:27.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:47.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:49.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:51.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:18.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:20.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:41.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:44.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:46.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:12.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:16.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:26.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:46.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:48.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:50.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:13.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:17.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:19.624] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:42.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:44.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:46.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:14.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:16.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:26.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:44.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:48.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:50.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:13.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:17.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:20.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:13.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:17.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:14.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:44.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:13.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:42.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:44.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:46.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:12.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:45.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:49.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:14.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:18.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:20.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:41.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:14.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:19.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:26.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:46.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:48.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:50.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:13.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:17.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:19.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:42.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:12.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:16.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:26.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:46.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:48.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:50.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:15.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:17.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:42.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:44.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:14.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:16.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:26.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:46.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:48.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:50.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:15.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:17.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:19.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:42.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:44.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:46.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:14.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:16.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:26.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:46.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:48.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:50.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:15.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:17.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:19.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:42.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:44.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:14.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:16.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:26.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:44.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:48.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:50.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:16.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:18.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:20.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:46.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:13.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:17.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:19.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:42.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:44.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:46.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:14.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:16.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:26.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:44.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:48.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:50.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:15.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:17.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:19.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:41.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:43.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:45.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:12.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:15.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:25.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:46.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:48.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:50.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:15.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:17.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:19.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:42.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:44.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:46.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:12.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:16.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:26.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:48.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:50.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:13.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:17.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:19.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:42.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:44.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:46.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:12.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:16.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:44.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:50.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:15.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:18.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:20.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:16.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:15.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:40.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:14.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:19.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:46.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:13.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:17.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:12.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:16.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:45.098] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:51.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:18.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:39.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:43.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:45.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:12.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:16.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:26.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:44.369] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:48.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:50.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:15.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:17.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:19.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:42.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:44.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:46.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:14.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:26.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:44.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:48.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:50.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:13.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:17.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:19.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:42.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:44.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:46.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:14.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:16.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:26.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:47.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:49.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:51.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:14.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:18.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:20.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:41.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:43.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:45.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:14.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:16.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:26.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:46.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:48.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:50.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:13.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:17.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:19.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:40.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:44.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:46.337] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:12.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:16.615] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:26.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:44.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:48.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:50.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:12.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:16.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:18.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:15.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:17.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:42.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:44.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:46.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:12.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:16.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:26.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:44.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:48.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:15.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:17.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:19.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:42.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:44.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:46.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:14.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:16.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:47.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:49.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:51.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:16.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:18.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:20.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:41.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:43.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:45.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:12.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:16.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:26.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:46.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:48.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:50.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:13.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:17.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:19.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:42.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:44.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:46.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:12.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:16.492] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:26.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:45.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:47.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:49.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:13.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:15.816] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:17.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:48.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:50.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:53.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:20.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:33.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:35.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:42.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:44.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:46.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:14.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:16.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:26.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:46.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:49.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:51.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:14.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:18.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:20.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:41.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:43.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:45.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:14.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:19.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:26.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:46.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:48.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:50.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:15.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:17.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:19.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:41.180] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:43.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:45.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:12.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:16.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:26.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:46.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:48.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:50.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:15.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:17.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:19.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:42.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:44.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:46.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:14.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:21.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:26.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:46.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:48.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:50.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:13.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:17.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:19.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:42.037] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:44.046] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:46.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:14.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:16.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:26.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:44.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:48.532] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:50.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:13.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:17.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:19.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:42.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:44.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:46.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:12.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:16.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:26.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:46.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:48.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:50.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:15.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:18.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:20.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:14.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:19.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:44.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:15.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:17.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:40.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:43.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:16.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:15.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:19.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:12.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:16.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:26.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:47.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:49.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:13.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:17.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:19.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:42.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:44.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:46.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:12.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:16.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:26.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:46.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:48.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:50.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:17.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:19.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:42.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:44.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:46.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:12.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:16.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:46.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:48.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:50.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:15.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:17.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:20.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:45.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:47.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:49.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:12.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:16.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:18.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:42.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:44.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:46.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:26.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:46.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:15.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:40.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:14.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:26.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:45.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:14.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:18.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:20.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:45.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:12.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:16.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:26.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:46.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:48.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:50.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:15.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:17.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:19.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:42.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:44.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:46.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:14.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:16.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:26.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:46.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:48.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:50.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:13.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:17.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:19.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:41.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:43.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:45.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:15.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:17.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:27.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:45.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:49.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:51.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:16.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:18.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:20.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:40.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:44.047] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:46.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:12.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:16.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:26.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:44.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:48.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:50.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:13.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:17.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:19.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:42.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:44.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:46.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:14.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:16.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:26.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:44.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:48.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:50.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:15.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:18.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:20.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:14.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:16.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:46.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:17.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:19.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:42.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:44.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:46.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:14.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:16.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:47.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:14.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:18.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:41.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:43.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:45.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:14.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:16.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:26.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:44.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:48.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:50.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:13.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:17.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:19.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:42.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:14.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:16.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:26.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:46.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:48.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:50.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:13.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:42.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:44.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:46.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:14.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:24.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:26.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:47.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:49.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:51.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:16.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:18.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:20.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:41.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:43.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:45.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:12.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:16.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:26.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:48.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:50.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:13.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:17.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:42.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:44.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:46.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:12.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:16.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:26.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:46.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:48.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:50.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:12.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:16.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:18.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:44.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:40.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:43.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:45.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:12.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:16.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:26.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:48.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:50.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:15.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:19.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:44.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:46.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:15.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:25.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:46.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:48.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:50.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:15.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:17.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:19.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:40.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:44.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:46.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:14.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:16.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:26.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:46.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:48.514] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:50.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:13.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:17.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:19.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:42.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:44.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:46.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:12.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:16.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:26.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:44.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:50.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:15.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:18.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:20.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:14.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:24.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:44.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:13.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:17.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:14.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:21.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:26.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:45.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:14.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:17.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:19.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:42.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:44.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:46.039] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:14.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:16.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:26.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:44.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:48.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:50.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:15.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:17.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:19.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:42.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:44.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:46.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:14.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:19.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:26.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:44.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:48.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:50.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:13.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:18.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:20.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:41.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:14.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:16.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:26.562] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:46.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:48.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:50.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:13.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:17.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:19.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:44.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:46.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:12.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:16.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:26.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:16.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:18.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:20.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:41.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:43.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:45.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:14.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:19.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:26.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:46.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:48.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:50.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:13.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:17.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:19.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:40.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:44.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:46.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:12.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:26.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:46.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:48.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:15.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:19.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:42.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:44.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:46.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:14.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:16.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:26.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:47.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:49.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:51.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:14.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:18.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:20.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:41.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:43.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:45.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:12.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:16.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:26.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:45.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:47.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:49.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:12.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:18.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:42.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:44.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:46.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:12.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:16.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:26.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:44.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:48.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:50.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:16.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:18.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:20.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:44.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:46.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:50.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:19.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:26.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:47.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:51.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:16.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:20.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:40.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:42.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:44.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:12.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:16.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:26.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:44.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:48.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:50.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:13.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:17.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:19.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:42.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:44.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:46.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:12.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:16.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:46.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:48.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:50.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:15.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:17.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:20.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:47.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:49.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:11.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:15.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:17.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:49.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:51.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:53.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:34.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:36.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:38.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:40.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:44.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:46.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:12.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:16.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:26.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:46.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:48.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:50.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:15.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:17.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:19.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:42.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:44.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:46.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:14.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:19.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:26.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:47.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:49.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:51.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:16.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:18.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:20.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:41.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:43.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:45.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:12.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:16.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:26.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:48.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:50.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:13.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:17.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:19.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:42.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:44.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:46.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:14.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:19.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:26.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:44.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:47.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:49.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:14.900] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:16.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:18.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:16.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:48.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:50.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:15.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:44.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:46.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:12.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:16.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:26.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:45.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:47.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:49.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:12.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:16.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:18.785] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:14.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:16.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:47.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:49.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:51.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:14.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:18.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:20.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:45.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:12.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:16.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:26.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:46.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:48.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:50.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:13.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:17.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:42.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:44.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:46.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:14.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:16.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:26.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:46.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:48.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:50.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:13.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:17.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:42.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:44.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:46.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:12.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:16.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:26.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:47.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:49.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:51.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:16.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:18.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:20.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:41.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:43.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:45.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:14.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:16.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:26.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:46.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:48.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:50.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:13.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:17.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:19.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:42.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:44.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:46.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:12.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:16.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:26.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:46.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:48.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:50.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:13.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:19.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:42.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:44.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:14.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:27.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:47.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:49.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:51.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:18.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:20.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:39.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:44.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:46.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:14.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:16.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:26.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:46.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:48.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:50.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:15.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:17.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:19.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:42.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:44.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:46.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:14.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:16.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:26.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:46.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:48.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:50.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:13.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:17.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:19.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:46.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:50.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:13.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:19.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:19.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:15.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:17.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:42.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:44.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:46.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:14.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:19.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:26.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:45.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:16.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:18.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:20.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:41.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:43.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:45.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:14.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:16.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:26.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:48.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:50.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:13.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:17.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:40.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:44.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:46.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:12.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:16.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:26.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:46.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:48.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:50.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:13.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:17.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:19.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:42.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:44.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:46.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:14.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:16.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:26.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:47.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:49.180] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:51.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:16.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:18.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:20.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:39.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:43.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:45.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:14.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:26.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:44.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:48.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:50.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:15.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:17.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:19.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:42.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:44.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:46.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:14.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:16.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:26.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:44.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:48.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:50.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:15.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:17.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:19.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:15.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:20.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:27.031] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:47.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:49.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:51.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:16.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:18.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:20.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:42.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:44.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:46.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:14.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:16.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:26.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:46.467] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:48.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:50.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:17.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:19.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:40.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:44.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:46.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:14.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:16.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:44.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:50.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:15.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:17.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:20.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:14.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:21.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:44.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:48.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:50.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:15.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:17.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:19.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:42.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:44.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:46.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:12.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:16.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:26.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:44.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:48.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:50.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:13.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:17.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:19.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:40.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:44.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:46.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:12.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:16.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:26.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:47.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:49.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:51.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:16.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:18.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:20.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:41.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:43.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:45.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:12.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:15.098] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:25.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:45.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:47.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:49.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:14.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:16.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:18.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:42.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:12.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:16.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:46.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:50.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:15.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:17.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:19.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:40.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:44.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:46.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:14.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:16.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:26.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:45.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:49.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:51.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:16.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:18.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:20.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:41.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:43.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:45.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:14.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:24.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:26.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:45.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:47.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:49.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:14.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:16.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:18.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:40.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:44.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:46.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:14.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:16.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:26.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:46.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:48.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:50.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:16.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:18.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:20.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:44.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:48.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:50.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:17.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:41.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:12.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:44.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:44.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:46.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:12.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:45.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:16.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:18.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:41.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:43.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:45.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:14.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:16.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:26.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:44.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:48.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:50.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:15.505] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:42.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:44.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:46.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:14.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:19.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:26.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:44.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:48.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:50.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:15.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:17.908] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:19.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:40.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:44.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:46.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:14.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:24.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:26.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:47.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:49.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:51.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:14.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:18.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:20.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:41.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:43.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:45.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:12.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:16.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:26.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:46.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:48.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:50.473] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:13.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:17.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:19.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:42.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:44.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:46.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:12.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:16.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:26.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:46.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:48.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:50.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:15.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:17.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:19.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:40.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:44.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:15.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:19.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:26.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:46.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:48.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:50.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:15.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:17.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:19.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:41.094] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:43.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:45.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:14.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:16.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:26.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:44.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:48.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:50.708] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:15.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:17.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:19.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:40.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:44.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:46.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:12.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:16.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:26.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:45.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:49.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:51.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:14.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:18.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:20.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:41.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:43.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:45.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:12.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:15.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:25.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:45.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:47.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:49.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:12.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:16.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:18.485] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:42.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:44.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:46.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:12.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:16.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:26.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:46.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:48.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:50.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:15.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:17.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:19.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:42.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:46.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:14.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:17.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:27.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:47.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:49.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:51.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:16.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:18.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:20.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:41.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:43.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:46.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:12.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:16.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:26.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:46.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:48.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:50.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:15.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:17.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:19.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:42.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:44.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:46.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:14.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:16.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:26.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:44.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:48.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:50.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:15.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:17.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:19.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:15.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:25.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:46.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:48.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:11.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:15.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:17.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:46.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:50.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:52.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:47:34.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:47:36.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:47:38.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:12.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:19.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:46.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:40.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:12.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:16.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:26.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:46.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:48.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:50.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:15.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:17.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:19.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:40.045] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:44.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:46.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:12.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:16.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:26.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:44.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:48.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:50.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:13.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:17.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:19.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:40.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:44.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:12.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:16.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:26.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:44.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:48.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:50.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:16.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:18.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:20.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:15.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:40.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:19.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:48.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:50.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:13.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:17.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:19.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:42.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:44.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:46.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:47.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:14.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:18.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:39.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:43.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:45.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:14.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:21.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:44.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:48.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:50.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:15.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:19.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:42.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:44.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:46.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:12.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:16.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:46.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:50.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:15.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:17.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:19.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:42.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:44.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:16.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:26.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:47.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:49.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:51.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:18.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:20.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:41.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:43.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:45.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:14.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:16.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:26.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:44.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:48.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:50.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:13.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:17.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:19.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:40.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:46.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:14.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:16.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:26.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:44.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:48.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:15.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:17.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:19.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:40.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:44.712] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:46.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:14.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:16.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:27.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:49.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:51.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:15.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:17.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:19.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:42.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:44.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:46.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:14.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:16.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:26.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:44.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:48.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:50.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:13.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:17.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:19.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:42.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:44.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:46.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:14.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:16.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:26.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:46.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:48.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:50.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:14.044] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:18.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:20.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:41.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:43.748] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:45.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:15.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:17.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:46.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:50.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:17.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:14.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:26.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:45.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:49.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:14.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:20.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:41.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:43.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:14.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:19.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:26.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:46.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:48.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:50.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:13.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:17.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:19.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:42.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:44.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:46.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:12.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:16.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:26.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:46.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:50.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:14.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:16.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:18.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:45.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:47.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:49.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:12.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:14.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:16.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:45.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:49.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:51.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:32.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:34.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:36.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:41.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:43.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:45.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:14.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:23.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:25.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:43.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:47.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:49.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:11.489] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:15.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:48.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:50.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:52.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:33:19.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:33:36.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:33:38.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:14.708] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:19.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:26.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:46.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:48.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:50.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:16.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:18.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:20.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:14.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:16.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:18.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:42.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:44.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:46.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:17.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:45.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:51.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:16.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:18.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:41.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:43.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:45.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:12.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:16.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:48.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:50.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:13.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:17.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:19.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:42.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:44.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:46.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:44.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:48.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:50.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:13.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:17.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:19.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:42.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:44.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:46.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:14.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:23.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:25.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:44.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:48.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:50.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:13.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:17.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:19.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:42.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:44.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:46.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:12.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:16.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:26.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:44.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:48.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:50.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:15.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:17.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:19.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:42.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:44.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:46.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:14.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:16.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:26.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:45.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:47.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:49.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:13.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:17.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:41.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:43.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:45.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:12.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:15.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:25.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:43.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:47.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:49.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:11.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:15.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:17.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:46.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:49.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:51.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:31.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:33.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:35.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:42.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:44.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:46.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:14.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:19.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:26.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:46.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:48.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:50.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:15.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:17.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:19.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:42.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:44.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:14.711] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:16.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:26.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:46.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:48.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:50.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:16.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:18.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:20.053] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:15.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:25.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:43.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:47.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:49.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:11.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:15.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:17.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:48.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:50.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:52.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:01:29.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:01:36.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:01:38.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:12.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:16.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:46.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:50.802] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:15.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:19.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:42.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:14.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:19.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:26.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:45.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:49.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:51.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:18.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:20.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:41.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:43.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:45.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:12.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:16.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:26.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:44.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:48.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:50.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:13.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:17.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:19.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:42.326] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:44.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:46.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:12.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:16.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:26.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:44.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:50.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:14.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:16.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:18.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:41.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:14.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:45.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:47.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:49.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:13.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:15.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:48.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:50.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:52.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:20.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:36.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:38.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:40.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:44.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:46.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:14.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:16.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:26.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:46.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:50.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:13.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:42.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:14.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:16.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:26.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:45.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:49.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:51.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:18.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:20.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:41.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:43.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:45.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:14.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:16.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:26.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:46.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:48.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:50.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:13.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:17.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:19.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:42.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:44.326] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:46.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:12.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:16.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:26.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:45.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:47.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:49.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:11.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:15.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:49.079] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:51.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:53.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:17.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:35.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:37.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:42.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:44.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:46.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:12.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:16.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:26.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:46.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:48.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:50.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:16.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:18.047] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:20.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:41.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:44.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:19.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:15.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:17.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:19.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:42.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:44.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:46.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:14.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:19.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:26.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:47.118] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:49.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:51.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:20.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:45.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:14.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:16.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:26.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:46.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:48.422] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:50.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:13.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:17.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:19.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:40.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:44.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:46.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:12.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:16.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:26.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:46.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:48.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:50.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:15.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:17.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:19.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:42.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:44.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:46.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:12.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:16.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:26.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:47.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:49.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:51.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:16.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:18.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:20.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:41.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:43.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:45.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:13.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:15.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:25.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:45.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:47.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:49.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:13.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:17.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:48.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:50.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:52.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:30.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:35.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:37.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:39.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:44.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:46.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:14.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:16.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:26.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:46.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:48.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:50.517] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:13.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:17.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:19.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:42.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:44.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:14.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:19.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:26.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:44.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:47.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:49.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:11.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:16.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:18.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:41.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:43.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:45.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:14.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:19.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:26.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:45.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:47.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:49.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:11.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:15.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:17.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:48.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:50.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:52.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:47:15.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:47:36.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:47:38.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:12.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:16.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:44.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:48.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:50.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:13.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:18.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:20.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:40.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:12.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:46.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:48.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:15.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:45.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:18.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:45.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:14.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:16.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:50.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:13.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:17.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:40.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:12.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:16.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:26.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:46.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:48.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:50.815] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:15.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:19.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:40.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:44.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:46.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:12.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:16.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:26.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:47.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:49.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:51.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:14.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:18.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:20.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:41.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:43.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:45.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:12.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:26.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:44.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:48.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:50.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:15.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:17.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:19.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:42.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:44.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:46.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:14.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:24.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:26.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:46.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:48.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:50.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:13.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:16.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:18.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:41.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:43.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:45.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:44.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:15.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:17.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:42.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:44.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:46.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:14.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:16.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:26.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:46.624] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:48.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:50.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:13.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:17.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:19.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:42.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:44.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:46.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:14.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:16.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:26.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:45.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:48.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:50.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:15.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:17.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:19.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:39.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:43.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:45.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:14.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:24.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:26.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:46.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:48.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:50.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:13.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:17.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:19.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:42.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:44.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:46.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:14.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:26.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:48.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:50.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:13.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:17.908] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:19.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:42.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:44.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:46.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:14.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:16.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:26.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:46.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:48.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:50.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:13.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:17.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:19.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:40.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:44.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:46.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:14.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:19.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:26.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:44.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:48.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:50.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:15.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:17.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:19.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:42.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:44.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:46.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:12.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:16.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:26.748] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:45.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:47.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:49.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:13.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:17.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:19.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:41.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:43.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:45.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:12.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:18.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:25.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:43.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:47.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:49.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:12.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:16.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:18.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:42.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:44.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:46.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:12.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:16.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:26.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:46.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:48.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:50.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:15.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:17.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:42.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:44.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:46.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:14.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:16.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:26.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:45.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:49.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:51.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:18.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:20.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:41.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:43.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:45.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:12.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:16.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:26.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:46.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:48.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:50.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:13.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:17.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:19.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:42.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:44.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:46.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:12.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:16.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:26.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:44.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:48.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:50.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:15.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:19.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:40.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:12.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:27.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:47.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:49.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:51.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:16.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:18.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:20.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:41.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:43.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:46.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:14.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:16.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:26.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:44.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:48.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:50.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:13.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:17.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:19.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:42.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:44.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:46.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:12.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:16.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:26.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:46.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:48.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:50.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:15.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:17.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:19.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:15.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:17.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:41.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:12.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:16.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:40.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:12.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:16.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:26.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:46.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:48.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:50.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:13.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:17.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:19.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:40.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:44.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:46.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:12.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:16.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:26.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:46.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:48.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:50.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:13.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:17.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:19.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:40.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:12.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:16.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:26.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:44.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:48.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:50.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:15.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:18.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:20.039] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:19.076] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:46.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:40.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:15.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:17.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:40.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:43.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:45.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:13.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:17.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:27.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:47.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:49.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:51.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:14.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:18.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:20.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:42.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:44.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:46.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:14.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:19.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:26.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:46.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:48.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:50.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:13.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:17.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:19.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:42.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:44.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:14.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:16.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:26.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:46.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:48.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:50.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:15.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:18.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:20.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:14.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:16.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:26.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:13.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:17.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:19.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:17.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:42.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:12.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:47.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:49.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:51.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:16.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:18.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:12.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:16.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:50.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:15.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:42.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:44.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:46.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:14.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:19.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:26.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:44.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:48.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:50.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:13.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:17.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:42.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:44.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:46.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:14.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:16.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:26.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:45.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:49.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:51.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:14.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:18.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:20.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:39.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:43.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:45.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:14.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:16.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:26.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:46.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:48.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:50.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:15.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:17.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:40.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:44.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:46.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:12.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:16.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:26.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:46.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:48.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:50.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:15.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:17.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:19.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:42.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:12.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:16.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:27.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:46.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:48.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:50.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:13.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:17.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:19.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:42.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:44.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:46.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:12.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:16.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:26.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:46.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:48.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:50.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:13.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:17.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:19.711] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:42.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:44.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:46.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:14.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:19.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:26.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:46.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:48.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:50.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:16.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:18.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:20.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:41.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:43.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:45.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:44.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:46.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:13.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:17.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:19.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:12.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:16.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:45.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:20.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:41.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:43.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:45.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:14.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:16.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:26.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:46.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:48.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:50.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:13.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:17.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:19.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:40.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:14.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:45.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:47.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:49.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:14.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:16.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:18.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:46.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:12.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:44.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:42.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:45.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:48.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:50.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:13.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:17.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:19.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:40.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:44.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:46.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:12.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:16.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:26.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:46.489] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:48.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:50.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:15.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:17.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:19.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:42.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:44.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:46.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:14.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:16.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:26.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:44.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:48.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:50.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:13.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:18.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:20.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:44.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:44.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:40.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:44.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:46.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:13.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:15.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:25.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:44.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:50.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:15.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:17.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:19.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:40.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:44.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:46.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:12.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:16.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:26.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:44.482] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:48.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:50.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:13.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:17.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:19.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:42.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:44.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:46.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:14.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:19.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:26.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:44.859] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:50.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:13.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:18.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:20.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:46.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:13.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:13.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:17.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:40.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:44.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:46.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:12.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:16.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:26.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:45.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:49.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:51.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:16.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:18.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:20.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:39.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:43.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:45.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:12.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:26.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:44.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:48.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:50.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:13.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:17.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:19.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:40.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:44.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:46.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:12.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:16.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:26.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:46.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:48.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:50.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:13.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:17.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:19.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:40.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:44.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:46.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:14.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:24.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:26.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:47.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:51.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:14.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:18.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:20.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:41.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:43.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:45.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:14.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:16.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:26.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:48.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:50.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:13.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:17.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:19.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:40.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:44.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:46.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:14.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:19.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:26.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:44.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:48.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:50.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:15.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:17.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:19.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:41.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:12.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:15.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:25.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:43.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:47.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:49.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:11.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:17.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:47.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:51.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:53.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:31.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:35.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:37.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:42.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:44.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:12.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:16.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:26.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:46.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:48.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:50.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:13.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:18.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:20.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:44.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:17.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:19.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:42.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:44.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:46.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:12.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:16.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:26.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:46.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:48.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:50.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:13.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:17.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:19.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:40.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:44.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:46.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:14.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:16.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:45.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:49.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:51.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:16.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:18.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:20.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:41.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:43.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:45.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:12.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:16.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:26.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:46.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:48.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:50.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:15.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:17.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:19.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:42.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:44.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:46.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:12.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:16.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:26.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:46.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:48.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:50.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:13.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:16.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:18.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:42.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:44.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:46.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:16.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:26.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:47.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:49.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:51.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:16.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:18.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:20.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:40.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:42.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:44.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:14.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:16.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:26.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:46.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:48.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:50.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:13.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:17.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:19.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:42.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:44.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:46.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:12.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:16.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:26.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:46.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:48.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:50.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:16.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:18.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:20.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:44.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:12.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:16.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:26.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:44.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:48.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:13.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:17.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:42.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:44.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:46.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:12.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:16.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:26.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:47.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:49.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:51.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:16.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:18.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:20.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:39.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:43.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:45.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:12.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:15.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:25.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:43.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:47.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:49.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:12.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:16.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:18.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:42.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:44.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:46.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:12.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:16.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:26.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:46.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:48.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:50.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:15.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:42.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:44.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:46.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:14.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:24.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:26.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:45.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:48.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:50.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:15.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:17.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:19.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:42.044] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:44.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:46.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:12.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:16.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:26.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:46.512] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:48.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:50.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:13.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:17.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:19.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:40.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:44.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:46.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:12.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:16.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:26.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:46.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:48.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:50.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:16.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:18.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:20.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:14.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:19.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:46.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:48.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:50.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:19.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:41.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:43.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:45.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:14.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:16.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:26.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:48.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:50.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:13.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:17.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:19.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:40.626] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:44.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:46.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:14.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:19.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:26.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:45.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:49.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:51.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:18.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:20.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:41.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:43.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:45.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:12.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:16.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:26.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:46.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:48.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:50.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:14.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:16.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:18.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:40.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:43.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:45.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:14.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:16.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:26.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:44.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:48.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:50.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:18.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:20.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:41.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:43.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:45.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:15.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:17.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:44.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:46.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:14.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:20.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:25.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:45.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:47.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:49.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:11.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:15.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:17.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:46.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:50.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:52.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:20.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:34.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:36.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:40.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:44.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:14.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:19.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:26.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:46.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:48.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:50.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:13.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:18.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:20.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:14.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:19.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:26.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:46.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:15.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:17.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:14.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:19.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:26.471] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:44.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:48.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:50.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:15.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:17.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:42.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:44.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:46.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:14.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:16.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:26.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:47.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:49.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:51.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:14.126] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:18.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:20.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:41.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:43.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:45.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:12.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:15.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:25.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:44.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:46.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:48.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:13.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:15.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:17.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:48.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:50.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:52.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:17.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:35.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:37.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:39.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:43.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:45.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:12.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:15.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:25.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:45.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:47.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:49.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:11.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:15.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:17.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:48.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:50.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:52.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:13:14.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:13:37.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:13:39.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:12.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:44.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:15.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:17.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:14.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:26.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:47.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:49.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:16.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:18.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:41.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:43.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:45.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:12.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:16.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:26.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:44.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:50.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:13.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:17.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:40.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:12.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:16.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:26.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:46.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:48.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:50.815] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:15.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:17.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:42.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:44.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:46.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:12.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:16.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:26.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:45.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:51.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:14.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:18.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:20.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:39.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:43.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:45.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:12.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:16.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:26.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:44.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:48.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:50.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:13.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:17.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:19.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:40.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:44.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:46.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:12.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:16.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:26.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:46.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:48.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:50.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:13.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:17.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:19.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:40.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:14.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:13.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:13.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:15.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:25.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:46.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:48.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:50.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:15.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:17.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:19.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:42.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:44.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:46.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:12.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:16.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:26.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:44.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:48.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:50.530] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:13.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:17.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:19.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:42.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:44.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:46.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:12.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:16.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:26.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:46.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:48.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:50.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:13.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:18.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:20.031] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:48.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:50.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:45.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:14.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:15.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:14.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:26.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:45.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:49.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:51.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:16.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:20.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:39.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:43.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:12.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:16.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:26.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:44.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:48.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:50.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:13.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:19.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:40.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:44.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:46.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:14.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:16.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:26.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:46.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:48.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:50.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:15.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:17.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:19.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:40.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:44.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:46.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:14.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:16.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:25.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:44.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:48.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:50.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:15.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:17.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:19.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:42.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:44.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:46.079] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:12.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:16.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:26.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:44.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:48.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:50.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:13.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:17.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:19.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:42.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:44.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:46.462] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:14.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:16.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:26.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:46.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:48.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:50.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:16.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:18.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:20.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:41.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:43.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:45.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:44.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:45.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:15.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:25.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:43.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:47.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:49.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:11.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:15.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:48.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:50.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:52.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:17.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:34.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:36.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:42.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:44.482] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:46.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:12.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:16.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:26.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:46.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:48.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:50.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:16.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:18.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:20.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:41.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:43.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:45.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:12.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:16.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:26.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:47.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:49.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:14.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:16.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:18.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:40.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:44.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:46.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:12.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:16.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:26.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:44.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:48.746] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:50.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:13.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:17.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:19.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:40.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:44.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:46.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:14.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:16.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:26.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:47.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:49.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:51.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:16.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:18.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:20.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:41.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:43.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:45.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:14.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:16.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:26.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:46.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:48.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:50.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:13.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:17.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:19.530] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:42.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:44.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:46.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:14.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:16.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:26.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:46.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:48.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:50.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:13.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:17.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:19.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:42.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:44.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:46.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:14.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:16.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:26.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:45.124] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:49.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:51.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:13.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:17.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:19.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:42.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:44.037] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:46.045] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:12.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:16.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:26.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:46.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:48.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:50.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:15.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:17.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:19.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:42.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:44.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:46.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:14.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:16.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:26.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:46.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:48.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:50.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:15.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:18.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:20.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:12.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:16.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:18.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:40.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:44.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:46.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:17.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:19.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:14.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:26.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:45.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:49.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:51.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:14.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:20.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:41.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:43.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:14.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:16.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:48.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:50.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:13.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:17.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:19.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:42.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:44.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:46.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:12.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:16.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:50.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:15.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:17.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:19.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:42.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:44.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:16.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:26.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:47.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:49.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:51.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:16.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:18.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:20.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:39.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:43.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:45.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:14.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:24.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:26.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:46.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:48.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:50.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:13.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:17.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:19.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:40.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:44.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:46.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:14.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:16.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:26.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:46.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:48.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:50.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:13.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:17.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:19.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:15.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:20.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:27.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:47.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:49.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:51.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:16.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:18.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:20.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:39.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:44.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:46.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:14.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:16.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:26.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:48.505] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:50.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:15.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:17.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:19.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:40.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:44.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:46.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:12.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:16.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:26.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:46.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:48.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:50.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:13.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:17.993] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:20.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:44.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:50.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:19.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:42.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:44.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:46.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:12.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:16.462] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:26.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:46.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:48.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:50.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:15.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:17.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:19.802] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:42.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:44.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:46.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:14.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:16.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:26.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:45.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:49.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:51.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:16.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:18.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:20.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:39.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:43.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:45.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:12.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:16.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:26.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:46.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:48.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:50.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:13.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:17.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:19.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:42.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:44.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:46.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:12.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:16.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:26.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:45.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:47.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:49.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:13.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:15.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:17.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:46.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:50.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:53.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:32.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:34.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:36.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:40.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:44.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:46.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:12.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:16.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:26.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:46.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:48.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:50.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:16.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:18.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:20.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:19.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:40.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:44.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:46.180] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:12.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:16.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:26.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:44.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:47.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:49.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:13.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:15.787] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:17.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:46.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:49.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:51.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:23.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:35.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:37.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:40.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:44.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:46.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:14.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:16.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:26.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:44.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:50.802] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:13.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:17.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:42.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:44.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:46.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:14.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:19.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:26.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:45.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:48.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:50.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:13.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:17.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:19.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:42.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:44.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:46.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:12.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:16.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:26.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:44.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:48.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:50.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:13.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:17.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:19.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:42.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:44.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:46.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:14.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:16.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:26.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:46.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:48.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:50.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:14.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:18.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:20.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:13.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:17.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:40.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:45.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:12.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:16.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:13.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:17.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:12.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:47.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:14.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:39.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:43.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:45.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:12.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:16.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:26.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:46.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:48.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:50.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:13.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:17.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:19.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:42.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:44.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:46.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:12.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:16.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:26.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:46.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:48.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:50.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:13.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:17.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:19.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:42.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:44.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:46.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:12.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:16.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:26.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:47.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:51.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:16.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:18.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:20.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:41.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:43.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:45.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:12.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:26.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:44.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:48.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:50.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:13.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:17.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:19.596] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:40.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:44.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:46.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:14.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:16.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:26.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:46.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:48.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:50.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:15.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:17.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:19.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:46.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:13.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:17.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:27.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:47.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:49.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:51.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:18.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:20.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:41.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:44.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:46.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:12.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:16.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:26.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:46.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:48.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:50.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:13.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:17.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:19.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:42.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:12.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:16.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:26.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:46.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:48.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:50.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:15.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:17.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:20.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:41.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:15.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:25.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:43.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:47.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:49.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:11.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:15.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:17.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:46.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:50.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:52.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:32.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:34.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:36.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:39.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:43.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:45.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:14.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:16.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:26.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:46.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:48.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:50.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:14.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:16.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:18.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:42.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:44.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:46.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:12.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:16.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:26.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:46.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:48.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:50.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:15.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:18.020] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:20.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:12.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:16.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:26.081] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:50.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:12.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:16.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:18.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:42.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:44.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:46.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:14.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:26.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:45.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:47.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:49.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:11.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:15.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:17.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:48.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:50.037] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:52.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:31.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:33.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:35.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:42.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:44.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:46.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:14.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:16.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:26.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:45.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:48.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:50.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:13.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:17.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:19.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:41.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:43.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:45.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:12.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:16.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:26.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:44.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:48.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:50.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:12.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:18.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:42.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:44.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:46.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:14.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:16.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:26.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:46.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:48.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:50.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:14.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:18.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:20.081] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:41.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:43.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:45.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:46.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:13.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:19.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:42.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:44.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:46.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:12.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:16.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:26.464] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:46.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:48.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:50.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:15.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:17.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:19.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:40.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:44.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:46.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:12.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:16.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:26.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:44.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:49.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:51.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:16.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:18.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:20.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:39.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:43.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:45.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:12.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:16.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:26.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:44.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:48.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:50.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:12.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:16.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:18.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:40.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:44.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:46.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:12.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:16.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:26.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:46.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:48.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:50.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:13.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:17.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:19.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:42.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:44.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:46.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:12.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:16.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:26.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:47.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:51.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:16.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:18.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:20.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:41.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:43.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:45.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:12.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:26.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:46.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:48.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:50.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:15.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:17.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:19.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:42.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:44.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:46.326] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:12.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:16.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:26.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:46.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:48.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:50.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:15.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:17.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:19.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:40.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:44.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:46.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:14.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:16.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:26.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:45.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:51.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:14.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:18.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:20.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:41.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:43.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:45.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:12.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:16.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:26.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:46.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:48.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:50.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:13.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:17.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:19.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:42.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:44.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:46.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:12.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:16.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:26.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:46.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:48.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:50.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:15.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:17.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:19.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:15.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:41.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:43.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:45.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:14.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:16.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:48.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:50.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:15.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:17.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:26.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:47.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:51.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:16.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:18.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:20.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:39.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:43.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:45.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:13.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:18.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:25.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:45.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:47.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:49.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:13.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:15.530] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:48.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:50.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:52.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:17.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:35.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:37.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:39.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:43.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:45.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:14.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:19.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:26.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:44.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:48.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:50.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:13.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:17.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:19.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:40.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:46.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:14.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:16.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:26.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:44.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:48.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:16.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:18.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:46.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:14.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:44.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:13.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:19.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:40.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:14.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:26.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:49.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:18.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:20.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:41.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:43.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:12.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:26.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:46.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:48.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:50.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:13.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:17.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:19.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:40.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:44.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:12.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:16.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:26.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:44.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:48.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:50.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:15.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:17.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:19.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:42.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:44.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:46.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:14.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:16.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:27.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:45.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:49.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:51.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:16.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:18.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:20.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:39.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:12.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:16.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:26.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:46.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:48.473] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:50.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:13.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:17.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:19.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:42.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:44.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:46.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:12.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:16.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:26.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:44.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:48.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:50.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:13.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:17.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:19.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:44.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:15.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:17.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:44.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:13.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:17.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:44.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:46.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:12.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:47.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:49.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:16.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:18.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:14.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:19.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:26.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:44.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:48.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:50.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:13.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:17.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:19.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:42.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:12.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:16.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:26.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:46.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:48.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:50.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:13.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:17.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:19.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:42.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:44.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:46.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:14.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:16.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:26.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:45.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:49.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:51.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:16.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:18.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:20.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:39.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:43.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:45.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:14.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:16.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:26.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:46.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:48.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:50.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:15.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:17.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:19.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:42.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:44.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:46.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:12.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:16.624] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:26.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:44.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:48.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:50.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:13.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:16.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:18.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:46.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:14.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:47.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:49.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:13.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:17.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:48.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:50.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:52.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:17.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:34.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:36.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:40.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:44.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:46.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:14.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:16.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:26.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:46.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:48.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:50.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:16.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:18.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:20.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:41.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:43.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:45.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:16.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:15.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:17.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:42.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:44.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:46.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:14.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:16.456] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:26.476] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:46.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:48.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:50.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:13.746] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:17.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:19.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:40.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:44.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:46.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:12.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:16.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:47.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:49.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:51.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:16.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:18.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:20.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:39.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:43.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:45.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:12.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:18.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:25.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:43.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:47.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:49.337] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:11.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:15.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:17.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:46.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:50.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:52.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:33.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:35.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:37.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:39.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:43.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:45.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:12.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:16.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:26.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:46.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:48.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:50.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:13.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:17.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:19.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:42.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:44.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:46.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:12.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:16.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:26.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:46.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:48.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:50.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:13.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:17.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:19.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:42.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:44.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:46.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:14.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:16.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:26.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:45.081] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:49.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:51.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:16.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:18.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:20.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:41.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:43.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:45.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:12.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:16.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:26.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:44.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:48.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:50.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:13.473] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:17.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:19.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:40.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:44.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:46.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:12.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:16.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:26.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:44.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:48.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:50.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:15.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:19.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:40.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:44.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:46.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:14.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:16.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:26.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:47.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:49.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:51.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:16.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:18.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:20.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:41.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:43.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:45.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:12.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:26.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:48.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:50.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:13.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:17.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:19.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:42.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:44.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:46.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:14.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:16.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:26.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:44.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:48.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:50.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:14.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:16.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:18.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:12.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:16.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:26.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:44.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:17.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:41.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:44.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:13.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:17.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:12.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:16.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:26.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:45.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:16.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:18.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:20.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:41.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:43.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:45.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:12.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:16.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:26.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:44.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:48.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:50.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:14.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:16.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:18.532] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:42.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:14.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:16.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:46.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:48.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:50.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:13.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:17.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:46.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:19.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:12.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:46.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:48.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:50.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:13.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:19.883] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:40.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:44.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:46.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:14.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:26.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:47.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:49.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:51.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:16.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:20.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:40.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:42.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:44.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:14.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:19.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:26.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:46.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:48.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:50.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:15.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:17.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:19.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:42.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:44.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:46.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:14.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:16.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:26.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:46.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:48.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:50.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:13.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:18.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:20.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:12.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:16.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:48.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:13.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:17.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:19.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:41.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:43.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:45.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:14.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:26.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:46.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:48.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:50.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:13.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:17.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:19.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:42.626] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:44.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:46.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:12.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:16.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:26.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:45.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:49.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:51.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:16.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:18.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:20.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:41.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:43.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:45.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:12.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:16.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:26.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:46.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:48.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:50.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:13.484] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:17.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:19.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:42.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:44.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:46.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:12.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:16.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:26.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:44.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:48.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:50.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:13.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:17.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:19.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:40.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:44.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:46.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:14.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:16.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:26.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:46.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:48.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:50.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:13.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:17.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:19.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:42.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:44.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:46.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:12.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:16.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:26.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:44.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:48.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:50.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:15.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:17.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:19.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:42.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:44.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:46.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:12.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:16.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:26.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:44.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:48.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:50.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:15.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:17.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:19.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:41.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:43.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:45.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:12.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:16.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:26.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:46.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:48.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:50.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:13.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:17.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:19.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:41.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:43.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:45.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:12.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:16.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:26.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:46.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:48.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:50.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:13.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:17.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:19.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:40.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:44.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:46.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:14.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:16.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:26.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:47.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:49.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:51.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:16.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:18.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:20.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:41.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:43.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:45.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:12.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:16.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:26.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:46.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:48.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:50.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:15.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:17.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:19.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:40.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:44.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:46.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:12.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:16.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:26.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:44.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:48.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:50.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:15.900] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:17.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:19.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:42.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:44.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:46.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:14.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:16.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:27.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:47.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:49.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:51.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:14.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:18.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:20.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:41.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:43.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:45.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:14.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:16.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:26.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:46.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:48.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:50.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:15.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:17.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:19.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:42.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:44.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:46.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:14.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:16.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:26.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:44.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:48.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:50.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:15.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:17.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:19.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:15.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:17.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:27.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:45.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:49.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:51.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:16.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:18.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:20.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:41.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:44.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:46.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:14.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:19.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:26.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:44.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:48.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:50.505] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:15.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:17.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:19.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:40.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:44.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:46.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:12.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:15.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:25.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:45.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:47.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:49.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:11.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:15.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:18.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:39.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:43.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:45.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:14.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:18.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:25.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:43.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:46.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:48.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:11.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:15.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:17.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:48.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:50.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:52.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:24:27.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:24:36.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:24:38.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:12.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:16.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:26.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:45.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:47.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:49.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:14.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:16.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:18.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:41.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:43.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:45.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:46.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:17.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:41.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:14.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:15.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:42.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:26.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:45.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:14.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:18.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:20.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:41.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:43.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:12.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:16.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:26.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:48.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:50.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:15.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:17.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:19.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:40.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:44.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:46.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:14.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:19.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:26.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:46.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:15.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:17.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:19.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:42.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:44.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:46.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:14.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:16.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:26.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:45.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:49.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:51.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:16.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:18.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:20.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:41.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:43.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:45.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:14.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:16.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:26.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:44.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:48.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:50.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:15.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:17.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:19.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:40.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:44.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:46.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:14.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:16.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:26.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:44.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:48.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:50.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:13.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:15.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:17.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:47.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:51.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:53.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:43:18.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:43:37.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:43:39.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:15.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:17.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:42.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:44.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:46.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:14.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:19.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:26.464] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:46.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:48.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:50.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:13.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:17.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:19.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:42.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:44.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:46.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:14.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:16.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:26.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:45.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:49.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:51.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:14.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:18.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:20.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:39.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:43.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:45.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:13.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:15.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:25.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:43.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:47.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:49.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:11.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:15.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:17.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:48.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:50.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:52.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:52:17.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:52:36.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:52:38.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:14.610] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:16.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:26.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:44.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:50.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:15.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:17.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:19.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:42.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:44.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:12.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:27.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:46.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:48.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:50.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:13.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:17.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:19.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:41.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:43.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:45.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:12.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:16.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:26.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:44.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:48.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:50.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:15.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:17.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:19.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:42.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:44.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:46.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:14.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:16.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:26.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:45.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:49.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:51.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:16.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:18.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:20.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:41.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:43.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:45.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:12.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:15.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:25.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:43.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:47.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:49.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:11.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:15.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:17.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:46.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:49.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:51.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:19.787] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:33.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:35.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:42.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:44.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:46.078] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:14.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:24.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:26.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:46.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:48.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:50.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:13.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:17.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:19.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:42.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:44.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:46.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:14.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:16.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:26.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:46.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:48.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:50.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:16.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:18.046] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:20.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:15.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:12.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:16.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:26.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:46.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:48.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:50.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:13.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:17.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:42.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:44.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:46.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:14.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:16.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:26.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:45.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:49.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:51.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:14.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:18.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:20.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:41.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:43.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:45.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:12.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:16.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:26.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:44.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:48.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:50.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:13.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:17.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:19.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:41.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:43.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:45.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:12.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:16.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:26.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:46.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:48.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:50.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:13.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:17.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:19.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:42.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:44.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:46.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:16.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:26.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:45.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:49.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:51.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:18.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:20.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:41.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:43.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:45.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:12.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:16.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:26.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:44.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:48.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:50.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:13.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:17.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:19.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:42.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:44.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:46.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:12.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:16.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:26.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:44.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:48.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:15.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:17.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:19.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:40.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:44.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:46.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:14.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:16.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:27.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:47.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:49.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:51.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:14.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:18.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:20.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:41.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:43.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:45.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:12.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:16.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:26.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:46.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:48.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:50.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:13.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:17.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:19.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:40.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:44.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:46.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:14.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:19.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:26.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:44.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:48.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:50.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:15.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:17.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:19.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:14.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:19.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:46.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:48.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:50.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:13.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:17.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:19.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:41.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:43.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:45.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:14.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:16.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:26.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:46.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:48.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:50.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:13.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:17.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:19.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:40.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:44.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:46.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:12.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:16.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:26.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:45.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:49.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:51.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:18.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:20.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:39.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:43.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:45.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:12.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:16.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:26.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:46.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:48.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:50.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:13.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:17.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:19.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:42.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:44.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:46.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:12.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:16.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:26.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:44.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:48.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:13.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:17.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:19.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:40.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:44.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:46.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:14.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:16.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:26.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:45.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:49.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:51.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:14.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:18.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:41.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:43.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:45.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:13.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:25.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:43.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:47.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:49.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:14.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:16.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:18.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:40.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:44.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:46.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:12.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:16.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:26.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:46.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:48.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:50.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:16.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:18.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:20.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:16.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:44.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:15.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:19.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:40.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:43.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:45.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:12.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:16.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:26.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:48.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:50.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:13.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:17.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:19.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:40.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:44.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:46.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:12.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:16.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:26.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:47.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:51.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:16.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:18.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:20.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:41.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:43.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:45.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:12.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:16.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:26.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:44.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:48.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:50.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:13.513] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:17.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:19.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:40.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:44.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:12.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:16.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:26.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:44.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:48.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:50.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:13.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:17.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:19.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:40.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:44.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:46.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:14.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:16.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:26.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:47.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:49.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:51.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:16.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:18.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:20.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:39.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:12.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:16.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:26.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:46.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:48.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:50.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:13.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:17.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:19.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:42.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:44.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:46.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:12.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:16.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:26.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:46.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:48.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:50.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:12.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:16.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:18.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:41.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:43.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:45.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:44.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:40.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:45.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:12.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:46.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:13.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:17.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:42.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:44.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:12.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:46.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:50.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:15.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:17.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:19.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:40.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:44.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:46.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:12.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:16.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:26.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:46.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:48.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:50.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:13.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:17.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:19.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:40.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:44.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:46.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:14.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:16.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:26.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:44.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:48.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:50.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:16.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:18.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:20.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:19.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:13.369] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:17.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:42.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:44.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:46.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:12.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:16.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:26.467] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:44.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:48.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:50.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:13.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:17.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:42.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:44.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:46.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:12.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:16.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:26.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:47.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:49.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:51.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:16.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:18.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:20.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:39.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:42.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:44.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:13.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:15.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:25.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:43.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:47.422] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:49.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:13.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:15.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:17.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:48.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:50.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:52.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:26.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:33.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:35.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:41.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:43.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:45.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:14.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:16.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:26.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:45.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:47.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:49.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:11.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:15.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:47.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:50.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:52.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:17.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:34.224] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:36.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:42.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:46.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:14.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:19.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:26.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:44.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:48.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:50.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:16.053] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:18.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:20.098] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:41.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:43.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:45.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:14.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:46.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:13.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:19.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:47.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:18.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:20.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:14.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:16.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:46.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:48.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:50.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:15.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:17.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:19.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:42.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:44.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:46.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:26.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:46.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:48.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:50.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:13.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:17.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:40.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:44.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:46.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:14.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:18.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:25.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:45.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:47.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:49.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:14.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:16.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:18.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:14.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:19.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:26.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:48.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:50.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:15.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:19.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:40.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:44.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:46.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:14.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:18.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:25.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:46.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:50.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:13.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:17.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:19.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:42.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:44.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:46.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:12.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:16.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:26.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:46.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:48.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:50.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:13.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:17.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:19.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:42.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:44.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:12.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:16.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:26.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:44.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:48.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:50.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:15.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:18.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:20.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:45.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:47.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:49.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:11.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:15.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:46.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:50.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:52.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:24.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:35.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:37.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:39.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:43.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:45.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:12.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:16.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:26.124] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:44.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:48.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:50.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:13.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:17.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:19.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:42.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:44.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:46.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:12.476] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:16.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:26.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:46.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:48.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:50.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:15.787] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:17.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:19.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:40.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:44.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:46.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:12.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:16.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:26.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:47.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:49.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:51.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:14.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:18.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:20.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:39.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:43.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:45.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:14.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:16.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:26.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:46.328] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:48.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:50.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:12.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:16.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:18.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:42.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:44.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:46.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:12.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:16.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:26.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:44.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:48.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:50.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:13.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:17.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:19.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:40.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:44.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:15.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:17.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:27.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:47.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:49.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:51.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:14.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:18.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:20.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:40.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:44.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:46.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:12.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:16.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:26.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:44.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:48.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:50.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:15.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:17.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:19.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:40.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:44.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:46.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:12.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:16.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:26.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:44.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:48.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:50.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:13.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:18.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:20.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:15.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:25.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:43.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:47.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:49.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:13.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:15.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:46.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:50.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:52.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:07:34.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:07:36.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:07:38.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:12.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:26.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:46.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:48.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:13.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:17.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:19.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:40.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:44.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:46.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:12.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:16.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:26.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:45.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:49.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:51.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:14.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:18.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:20.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:41.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:43.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:45.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:14.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:24.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:26.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:48.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:50.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:13.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:17.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:19.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:40.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:44.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:46.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:12.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:16.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:26.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:44.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:50.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:15.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:17.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:19.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:40.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:44.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:46.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:12.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:27.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:49.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:51.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:14.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:17.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:19.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:40.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:44.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:46.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:12.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:16.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:26.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:45.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:47.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:49.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:12.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:16.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:18.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:42.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:44.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:46.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:14.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:19.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:26.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:47.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:49.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:51.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:16.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:18.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:20.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:41.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:43.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:45.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:13.119] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:15.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:25.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:43.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:47.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:49.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:12.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:16.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:18.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:40.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:44.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:46.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:14.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:16.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:26.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:46.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:48.815] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:50.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:15.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:17.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:19.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:42.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:44.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:46.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:14.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:16.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:26.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:47.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:49.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:51.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:16.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:18.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:20.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:39.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:43.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:45.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:12.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:16.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:26.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:46.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:48.462] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:50.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:13.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:17.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:19.596] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:42.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:44.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:46.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:12.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:16.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:26.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:46.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:48.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:50.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:15.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:17.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:19.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:42.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:44.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:46.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:13.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:17.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:27.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:45.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:49.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:51.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:18.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:20.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:41.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:44.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:46.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:12.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:16.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:26.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:46.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:48.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:50.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:17.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:19.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:40.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:46.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:14.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:16.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:26.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:46.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:48.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:50.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:15.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:17.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:20.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:47.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:49.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:11.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:15.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:17.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:46.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:50.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:52.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:29.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:34.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:36.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:41.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:43.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:45.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:14.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:19.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:26.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:44.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:48.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:50.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:14.492] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:16.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:18.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:40.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:46.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:14.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:16.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:44.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:48.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:50.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:15.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:17.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:44.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:46.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:48.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:50.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:17.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:40.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:12.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:16.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:47.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:49.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:51.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:14.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:18.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:20.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:39.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:43.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:45.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:12.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:26.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:46.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:48.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:50.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:13.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:17.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:19.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:42.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:44.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:14.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:16.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:26.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:46.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:48.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:50.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:15.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:17.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:19.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:40.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:44.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:46.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:14.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:16.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:27.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:45.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:49.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:51.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:16.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:18.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:20.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:41.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:12.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:16.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:26.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:46.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:48.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:50.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:13.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:17.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:19.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:40.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:44.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:46.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:12.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:16.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:26.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:45.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:47.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:49.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:13.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:15.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:17.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:39.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:43.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:45.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:13.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:15.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:25.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:43.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:47.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:49.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:11.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:15.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:17.464] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:48.596] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:50.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:52.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:03:14.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:03:36.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:03:38.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:14.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:16.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:26.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:46.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:48.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:50.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:13.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:17.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:19.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:42.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:46.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:15.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:17.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:27.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:45.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:49.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:51.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:16.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:18.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:20.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:39.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:44.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:46.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:12.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:16.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:26.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:46.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:48.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:50.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:15.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:17.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:19.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:42.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:44.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:46.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:12.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:16.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:26.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:44.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:48.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:50.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:13.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:17.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:20.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:40.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:43.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:45.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:14.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:24.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:26.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:44.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:48.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:50.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:15.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:19.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:40.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:44.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:46.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:14.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:16.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:26.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:47.120] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:51.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:16.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:18.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:20.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:41.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:43.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:45.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:14.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:19.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:26.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:44.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:48.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:50.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:13.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:17.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:19.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:42.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:44.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:14.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:16.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:26.615] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:46.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:48.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:50.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:15.900] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:17.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:19.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:42.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:44.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:46.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:12.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:16.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:26.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:47.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:49.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:51.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:14.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:18.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:20.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:41.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:12.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:16.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:26.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:46.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:48.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:50.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:15.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:17.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:19.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:40.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:44.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:46.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:14.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:16.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:26.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:45.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:47.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:49.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:11.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:15.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:17.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:41.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:43.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:45.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:12.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:16.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:26.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:48.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:50.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:13.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:17.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:19.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:42.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:44.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:46.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:12.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:16.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:26.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:48.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:50.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:15.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:17.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:19.851] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:40.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:44.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:46.615] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:12.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:16.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:26.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:45.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:49.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:51.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:16.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:18.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:20.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:41.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:43.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:45.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:12.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:16.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:26.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:44.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:48.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:50.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:15.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:17.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:19.514] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:40.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:44.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:46.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:14.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:16.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:26.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:46.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:48.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:50.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:13.851] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:17.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:19.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:40.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:44.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:46.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:12.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:16.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:26.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:45.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:50.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:13.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:17.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:19.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:42.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:44.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:46.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:12.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:16.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:26.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:44.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:48.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:50.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:13.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:17.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:19.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:42.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:44.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:46.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:12.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:16.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:26.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:46.883] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:48.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:50.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:16.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:18.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:20.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:44.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:15.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:17.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:19.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:42.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:44.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:46.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:12.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:16.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:26.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:46.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:48.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:15.748] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:17.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:19.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:42.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:44.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:46.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:12.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:16.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:26.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:47.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:49.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:51.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:14.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:18.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:20.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:41.816] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:43.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:45.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:14.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:19.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:26.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:46.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:48.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:50.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:14.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:16.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:18.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:42.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:44.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:46.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:12.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:16.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:26.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:46.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:50.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:42.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:44.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:12.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:16.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:26.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:47.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:49.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:51.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:16.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:18.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:20.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:39.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:43.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:45.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:12.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:16.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:26.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:44.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:48.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:50.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:15.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:17.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:19.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:40.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:44.337] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:46.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:12.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:16.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:26.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:45.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:47.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:49.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:11.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:15.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:17.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:47.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:51.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:53.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:18.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:36.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:38.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:40.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:44.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:46.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:12.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:16.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:26.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:44.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:48.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:50.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:15.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:17.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:19.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:12.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:16.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:13.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:17.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:41.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:13.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:17.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:40.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:14.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:16.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:26.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:47.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:49.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:51.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:16.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:18.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:20.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:41.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:43.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:45.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:26.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:46.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:48.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:50.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:13.513] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:19.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:40.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:44.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:46.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:14.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:16.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:26.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:46.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:48.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:50.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:12.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:16.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:18.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:13.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:17.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:24.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:46.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:48.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:15.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:17.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:46.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:50.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:52.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:23.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:35.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:37.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:41.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:43.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:45.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:12.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:16.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:26.124] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:46.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:48.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:50.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:15.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:17.422] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:19.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:40.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:44.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:46.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:14.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:16.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:26.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:44.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:48.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:50.708] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:15.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:17.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:19.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:40.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:44.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:46.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:14.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:21.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:26.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:47.047] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:49.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:51.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:16.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:18.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:20.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:39.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:43.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:45.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:12.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:16.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:26.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:44.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:48.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:50.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:13.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:17.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:19.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:40.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:44.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:46.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:14.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:16.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:26.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:48.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:50.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:12.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:16.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:18.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:40.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:43.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:45.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:12.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:16.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:26.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:46.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:48.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:50.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:13.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:19.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:40.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:44.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:46.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:14.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:16.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:26.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:47.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:51.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:13.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:17.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:19.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:42.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:44.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:46.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:12.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:16.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:26.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:46.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:48.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:50.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:13.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:17.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:19.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:40.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:44.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:46.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:12.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:16.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:26.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:44.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:48.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:50.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:15.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:18.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:20.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:44.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:14.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:16.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:18.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:42.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:44.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:46.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:14.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:16.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:26.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:48.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:50.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:15.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:17.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:19.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:42.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:44.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:46.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:12.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:16.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:26.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:51.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:15.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:17.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:19.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:42.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:44.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:46.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:14.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:26.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:46.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:48.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:50.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:13.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:17.626] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:19.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:42.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:44.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:12.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:16.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:26.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:44.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:48.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:50.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:13.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:17.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:20.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:15.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:25.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:45.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:47.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:49.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:11.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:15.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:48.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:50.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:52.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:48:19.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:48:37.746] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:48:39.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:14.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:16.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:26.476] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:44.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:48.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:50.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:15.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:17.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:19.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:40.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:44.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:46.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:14.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:16.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:26.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:46.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:48.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:50.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:13.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:17.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:19.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:41.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:43.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:45.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:14.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:20.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:25.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:43.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:47.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:49.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:11.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:15.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:17.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:46.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:50.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:52.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:32.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:34.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:36.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:42.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:44.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:46.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:14.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:16.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:26.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:44.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:48.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:50.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:13.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:17.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:19.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:40.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:44.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:46.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:12.712] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:16.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:26.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:44.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:48.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:50.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:14.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:18.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:20.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:44.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:40.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:14.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:50.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:15.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:12.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:26.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:45.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:14.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:18.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:41.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:43.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:12.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:26.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:46.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:48.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:50.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:15.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:17.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:19.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:42.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:44.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:46.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:12.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:16.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:26.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:46.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:48.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:50.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:15.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:17.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:40.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:44.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:46.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:14.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:16.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:26.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:47.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:49.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:51.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:16.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:18.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:20.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:41.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:43.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:45.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:12.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:16.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:26.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:44.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:48.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:50.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:13.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:17.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:19.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:40.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:44.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:46.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:12.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:16.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:26.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:46.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:48.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:50.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:15.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:17.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:19.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:13.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:17.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:27.031] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:45.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:49.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:51.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:14.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:18.328] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:20.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:41.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:44.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:46.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:14.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:16.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:26.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:44.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:48.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:50.513] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:13.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:17.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:19.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:40.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:44.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:46.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:14.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:16.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:45.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:47.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:49.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:11.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:15.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:18.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:41.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:43.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:45.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:12.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:16.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:26.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:44.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:48.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:50.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:14.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:16.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:18.489] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:40.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:44.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:46.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:14.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:16.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:26.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:44.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:48.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:50.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:15.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:17.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:19.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:40.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:44.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:46.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:12.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:17.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:27.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:47.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:49.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:51.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:14.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:18.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:20.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:41.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:44.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:46.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:12.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:16.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:26.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:46.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:48.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:50.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:17.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:19.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:40.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:44.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:46.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:12.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:16.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:26.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:44.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:48.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:50.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:15.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:17.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:20.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:13.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:17.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:19.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:44.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:46.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:14.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:16.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:26.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:46.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:48.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:50.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:15.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:17.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:19.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:42.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:44.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:46.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:14.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:19.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:45.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:49.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:51.053] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:14.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:18.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:20.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:41.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:43.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:45.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:12.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:16.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:26.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:46.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:48.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:50.328] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:15.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:17.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:19.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:42.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:44.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:46.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:12.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:19.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:26.512] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:46.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:48.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:50.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:13.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:17.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:19.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:40.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:44.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:46.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:14.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:16.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:26.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:47.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:49.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:51.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:14.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:18.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:20.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:41.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:43.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:45.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:13.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:15.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:25.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:45.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:47.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:49.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:11.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:15.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:17.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:48.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:50.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:52.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:46:17.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:46:36.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:46:38.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:12.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:16.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:26.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:44.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:48.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:50.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:15.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:18.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:20.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:46.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:48.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:50.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:17.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:41.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:17.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:14.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:16.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:26.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:47.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:49.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:51.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:18.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:20.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:39.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:43.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:45.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:14.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:16.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:26.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:44.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:48.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:50.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:15.492] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:17.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:19.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:42.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:44.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:46.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:14.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:19.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:26.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:46.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:48.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:50.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:15.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:17.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:19.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:42.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:44.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:46.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:12.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:16.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:26.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:45.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:49.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:51.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:14.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:18.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:39.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:43.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:45.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:12.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:16.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:26.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:46.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:48.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:50.456] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:15.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:17.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:19.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:42.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:44.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:46.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:14.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:19.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:26.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:44.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:13.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:19.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:40.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:44.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:14.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:20.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:27.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:47.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:49.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:51.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:16.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:18.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:20.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:39.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:43.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:46.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:14.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:16.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:26.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:46.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:48.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:50.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:13.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:17.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:19.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:42.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:44.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:46.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:12.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:16.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:26.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:46.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:48.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:50.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:13.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:17.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:19.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:13.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:15.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:25.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:45.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:47.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:49.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:11.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:15.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:17.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:46.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:50.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:52.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:13:34.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:13:36.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:13:38.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:12.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:16.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:46.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:50.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:13.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:17.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:19.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:42.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:44.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:46.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:12.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:16.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:26.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:47.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:49.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:51.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:16.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:18.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:20.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:41.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:43.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:45.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:12.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:16.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:26.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:48.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:50.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:13.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:17.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:19.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:40.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:44.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:46.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:12.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:16.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:26.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:46.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:48.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:15.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:17.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:19.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:40.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:44.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:46.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:14.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:16.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:27.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:49.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:51.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:14.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:18.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:20.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:39.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:43.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:46.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:12.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:16.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:26.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:45.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:47.484] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:49.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:11.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:15.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:17.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:48.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:52.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:26:19.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:26:37.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:26:41.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:14.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:16.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:26.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:44.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:48.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:50.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:15.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:18.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:20.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:40.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:12.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:16.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:26.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:46.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:48.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:15.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:17.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:19.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:40.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:44.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:46.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:12.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:16.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:26.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:45.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:49.039] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:51.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:14.120] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:18.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:20.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:41.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:43.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:45.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:12.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:16.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:26.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:44.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:48.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:50.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:13.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:17.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:19.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:42.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:44.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:46.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:12.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:16.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:26.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:46.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:48.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:50.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:15.785] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:17.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:19.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:40.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:44.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:46.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:14.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:19.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:26.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:47.045] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:49.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:51.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:14.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:18.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:20.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:39.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:43.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:45.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:12.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:15.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:25.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:44.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:46.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:48.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:11.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:15.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:17.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:47.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:49.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:51.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:19.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:33.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:35.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:40.078] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:44.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:46.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:12.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:16.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:26.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:46.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:48.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:50.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:13.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:17.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:19.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:42.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:46.482] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:14.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:19.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:26.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:44.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:48.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:50.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:16.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:18.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:20.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:41.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:43.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:45.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:48.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:50.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:14.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:16.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:18.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:40.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:44.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:46.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:12.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:16.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:26.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:13.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:42.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:12.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:16.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:47.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:49.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:51.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:14.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:20.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:45.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:12.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:16.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:26.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:46.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:50.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:13.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:17.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:19.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:40.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:44.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:46.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:14.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:19.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:26.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:44.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:48.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:50.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:15.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:17.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:42.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:44.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:46.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:12.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:16.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:26.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:45.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:48.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:50.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:15.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:17.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:19.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:40.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:44.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:46.076] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:12.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:16.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:26.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:46.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:48.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:50.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:13.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:17.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:19.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:40.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:44.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:46.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:14.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:16.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:26.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:44.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:48.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:50.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:14.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:18.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:20.076] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:41.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:15.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:42.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:44.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:46.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:12.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:16.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:26.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:45.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:47.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:49.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:11.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:15.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:17.785] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:46.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:50.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:52.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:31.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:34.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:36.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:40.378] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:12.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:16.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:44.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:48.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:50.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:15.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:18.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:20.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:13.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:40.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:12.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:44.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:48.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:15.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:17.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:12.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:26.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:45.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:49.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:51.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:15.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:17.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:19.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:40.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:44.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:46.020] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:14.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:16.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:26.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:46.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:48.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:50.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:15.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:17.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:19.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:40.378] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:14.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:19.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:26.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:44.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:48.883] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:50.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:15.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:17.993] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:20.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:19.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:40.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:44.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:46.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:12.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:16.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:26.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:47.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:49.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:51.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:14.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:18.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:20.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:41.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:43.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:45.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:14.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:16.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:26.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:45.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:47.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:49.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:11.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:15.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:17.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:46.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:50.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:52.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:29:19.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:29:36.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:29:38.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:14.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:16.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:26.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:44.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:48.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:50.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:16.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:18.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:20.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:41.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:43.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:45.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:44.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:15.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:17.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:40.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:14.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:16.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:15.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:17.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:40.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:45.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:16.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:18.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:20.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:41.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:43.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:45.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:12.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:16.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:26.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:44.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:47.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:49.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:11.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:15.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:17.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:48.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:50.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:52.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:33.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:35.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:37.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:39.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:43.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:45.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:12.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:25.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:45.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:47.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:49.467] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:13.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:15.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:17.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:46.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:50.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:52.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:43:17.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:43:36.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:43:38.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:14.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:19.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:26.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:44.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:48.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:50.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:14.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:18.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:20.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:41.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:43.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:45.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:46.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:48.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:50.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:14.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:16.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:18.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:40.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:44.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:46.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:12.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:16.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:26.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:46.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:48.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:50.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:13.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:19.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:42.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:44.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:46.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:14.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:16.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:26.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:45.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:49.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:51.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:14.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:18.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:20.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:39.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:43.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:45.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:14.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:16.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:26.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:46.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:48.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:50.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:13.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:17.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:19.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:42.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:44.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:46.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:12.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:16.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:26.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:44.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:48.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:50.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:12.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:18.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:40.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:46.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:48.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:50.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:15.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:42.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:44.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:46.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:14.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:16.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:26.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:45.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:49.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:51.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:16.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:18.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:20.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:45.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:12.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:15.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:25.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:43.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:47.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:49.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:13.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:15.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:17.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:48.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:50.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:52.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:04:14.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:04:37.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:04:39.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:13.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:15.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:25.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:43.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:47.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:49.816] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:14.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:16.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:18.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:12.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:44.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:15.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:14.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:19.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:45.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:20.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:41.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:43.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:45.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:26.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:48.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:50.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:15.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:17.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:19.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:42.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:44.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:46.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:14.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:16.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:26.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:46.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:48.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:50.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:15.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:19.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:40.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:44.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:46.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:14.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:19.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:26.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:45.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:49.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:51.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:16.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:18.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:20.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:41.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:43.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:45.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:12.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:16.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:26.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:46.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:48.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:50.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:13.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:17.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:19.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:42.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:44.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:46.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:12.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:16.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:26.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:44.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:48.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:50.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:13.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:17.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:19.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:42.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:46.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:14.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:21.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:26.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:46.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:48.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:50.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:13.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:17.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:19.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:41.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:43.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:45.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:12.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:16.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:26.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:44.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:48.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:50.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:13.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:17.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:19.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:42.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:44.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:46.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:12.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:16.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:26.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:47.046] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:49.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:51.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:16.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:18.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:20.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:39.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:43.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:45.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:12.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:16.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:26.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:46.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:48.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:50.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:13.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:17.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:19.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:42.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:44.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:46.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:14.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:16.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:26.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:46.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:48.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:50.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:13.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:17.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:19.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:42.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:44.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:46.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:12.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:16.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:26.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:47.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:49.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:51.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:14.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:18.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:20.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:41.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:43.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:45.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:12.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:16.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:26.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:46.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:48.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:50.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:13.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:17.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:19.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:42.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:44.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:46.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:44.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:50.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:12.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:16.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:18.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:13.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:46.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:17.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:19.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:45.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:49.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:14.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:14.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:16.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:44.378] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:48.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:50.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:13.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:17.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:19.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:42.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:44.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:46.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:12.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:16.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:26.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:46.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:48.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:50.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:15.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:19.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:42.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:44.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:46.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:14.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:19.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:26.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:45.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:49.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:51.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:16.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:18.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:41.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:43.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:45.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:12.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:16.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:26.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:44.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:48.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:50.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:15.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:17.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:19.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:42.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:46.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:12.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:16.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:26.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:44.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:48.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:50.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:15.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:17.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:19.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:42.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:12.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:17.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:27.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:47.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:49.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:51.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:16.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:18.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:20.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:41.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:44.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:46.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:12.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:16.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:26.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:46.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:48.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:50.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:15.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:17.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:19.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:40.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:44.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:46.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:12.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:16.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:26.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:44.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:47.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:49.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:13.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:15.993] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:18.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:41.851] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:43.859] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:45.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:12.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:16.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:26.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:46.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:48.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:50.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:13.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:17.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:19.485] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:42.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:46.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:14.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:16.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:26.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:44.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:48.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:50.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:12.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:16.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:18.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:40.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:44.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:46.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:14.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:17.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:27.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:46.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:48.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:50.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:13.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:17.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:19.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:42.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:44.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:46.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:12.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:16.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:26.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:44.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:48.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:50.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:13.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:17.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:19.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:40.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:44.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:46.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:14.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:18.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:25.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:45.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:47.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:49.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:13.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:17.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:19.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:41.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:43.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:45.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:12.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:16.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:26.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:44.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:48.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:50.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:13.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:17.456] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:19.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:42.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:44.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:46.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:12.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:16.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:26.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:46.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:48.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:50.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:12.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:16.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:18.859] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:40.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:14.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:27.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:47.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:49.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:51.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:14.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:17.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:19.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:41.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:43.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:45.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:12.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:16.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:26.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:44.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:48.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:50.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:13.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:17.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:19.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:42.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:44.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:46.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:14.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:19.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:26.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:47.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:49.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:51.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:16.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:18.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:20.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:41.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:43.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:45.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:13.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:15.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:25.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:43.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:46.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:48.369] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:13.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:15.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:17.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:46.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:50.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:52.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:23:19.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:23:36.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:23:38.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:12.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:16.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:44.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:48.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:50.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:12.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:16.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:19.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:41.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:43.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:45.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:16.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:15.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:40.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:14.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:44.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:48.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:50.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:13.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:42.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:44.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:12.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:26.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:47.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:16.224] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:45.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:14.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:24.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:46.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:48.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:50.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:13.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:19.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:42.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:44.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:46.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:26.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:13.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:19.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:40.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:14.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:16.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:26.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:45.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:48.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:50.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:13.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:17.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:19.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:42.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:44.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:46.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:14.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:16.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:26.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:44.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:48.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:50.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:12.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:16.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:18.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:42.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:44.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:46.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:12.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:16.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:26.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:46.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:49.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:51.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:15.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:17.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:19.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:41.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:43.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:45.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:12.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:16.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:26.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:46.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:48.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:50.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:13.485] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:17.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:19.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:42.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:44.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:46.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:14.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:16.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:26.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:46.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:48.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:15.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:19.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:40.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:44.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:46.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:14.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:16.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:26.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:45.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:49.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:51.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:16.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:18.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:41.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:43.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:45.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:12.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:16.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:26.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:44.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:48.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:50.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:13.532] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:17.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:19.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:42.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:46.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:12.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:16.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:26.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:46.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:48.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:12.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:18.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:12.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:16.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:26.084] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:50.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:15.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:17.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:19.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:40.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:43.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:45.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:14.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:16.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:48.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:50.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:15.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:40.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:19.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:47.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:51.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:16.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:18.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:20.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:39.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:43.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:12.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:16.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:26.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:46.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:48.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:50.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:15.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:19.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:40.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:14.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:18.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:29.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:43.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:45.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:47.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:17.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:21.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:23.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:42.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:44.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:46.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:12.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:16.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:26.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:46.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:48.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:50.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:15.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:17.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:14.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:16.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:18.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:42.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:44.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:46.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:14.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:19.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:44.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:15.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:17.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:26.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:51.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:16.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:12.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:16.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:26.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:44.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:48.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:50.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:15.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:19.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:23.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:42.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:46.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:50.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:12.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:16.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:23.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:42.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:46.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:50.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:16.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:31.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:33.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:39.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:43.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:54.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:11.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:15.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:21.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:43.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:45.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:49.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:16.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:25.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:38.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:42.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:46.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:12.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:20.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:28.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:48.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:52.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:56.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:16.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:20.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:24.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:47.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:04.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:06.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:15.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:23.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:27.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:41.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:45.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:50.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:16.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:25.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:29.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:53.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:01.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:03.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:14.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:22.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:26.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:42.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:46.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:48.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:21:23.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:21:40.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:21:44.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:14.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:18.562] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:30.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:42.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:46.712] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:50.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:32.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:36.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:40.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:44.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:48.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:53.546] error: *** JOB 14360421 ON gpua094 CANCELLED AT 2025-12-28T04:23:53 DUE to SIGNAL Terminated *** +[2025-12-28T04:24:01.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:24:17.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:24:21.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:24:26.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:26:24.001] error: namespace_p_join: open failed for /local/slurmjobs/14360421/.ns: No such file or directory +[2025-12-28T04:26:24.001] error: namespace_g_join(14360421): No such file or directory +s-nail: Cannot find a usable character set to encode message: No such entry, file or directory +s-nail: ... message not sent +[2025-12-28T04:26:25.000] error: *** JOB 14360421 STEPD TERMINATED ON gpua094 AT 2025-12-28T04:26:23 DUE TO JOB NOT ENDING WITH SIGNALS *** +[2025-12-28T04:26:33.135] error: Container 2393516 in cgroup plugin has 2 processes, giving up after 127 sec diff --git a/collaborativeagents/slurm/logs/rag_vector_14360422.err b/collaborativeagents/slurm/logs/rag_vector_14360422.err new file mode 100644 index 0000000..8195f29 --- /dev/null +++ b/collaborativeagents/slurm/logs/rag_vector_14360422.err @@ -0,0 +1,129 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:59:04,500 - INFO - Loaded dataset: mmlu +2025-12-27 01:59:04,500 - INFO - Loaded dataset: aime +2025-12-27 01:59:04,500 - INFO - Loaded dataset: math-hard +2025-12-27 01:59:04,500 - INFO - Loaded dataset: humaneval +2025-12-27 01:59:04,516 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:59:04,517 - INFO - Running method: rag_vector +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.76s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.40s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.15s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.67s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.97s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.84s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.56s/it] +2025-12-27 01:59:47,035 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:08<01:04, 8.09s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:18<01:07, 9.62s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:29<01:00, 10.11s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:40<00:51, 10.29s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:56<00:49, 12.41s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:06<00:35, 11.72s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:15<00:21, 10.95s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:22<00:09, 9.70s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:25<00:00, 7.58s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:25<00:00, 9.55s/it] +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-27 02:05:38,040 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:26:36,031 - WARNING - User agent failed to respond at turn 2 +2025-12-27 02:49:12,707 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:49:16,867 - INFO - Profile 2/30 +2025-12-27 03:15:00,486 - WARNING - User agent failed to respond at turn 3 +2025-12-27 03:26:15,013 - INFO - Profile 3/30 +2025-12-27 04:03:36,467 - WARNING - User agent failed to respond at turn 2 +2025-12-27 04:08:15,791 - INFO - Profile 4/30 +2025-12-27 04:22:36,794 - WARNING - User agent failed to respond at turn 2 +2025-12-27 04:25:15,739 - WARNING - User agent failed to respond at turn 3 +2025-12-27 05:08:15,379 - INFO - Profile 5/30 +2025-12-27 05:45:53,211 - INFO - Profile 6/30 +2025-12-27 06:37:29,994 - INFO - Profile 7/30 +2025-12-27 06:44:34,982 - WARNING - User agent failed to respond at turn 5 +2025-12-27 07:07:39,808 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:24:40,817 - WARNING - User agent failed to respond at turn 2 +2025-12-27 07:40:10,779 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:40:10,781 - INFO - Profile 8/30 +2025-12-27 07:42:53,487 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:59:51,049 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:20:46,161 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:37:34,229 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:37:34,231 - INFO - Profile 9/30 +2025-12-27 09:21:28,395 - INFO - Profile 10/30 +2025-12-27 09:47:44,540 - WARNING - User agent failed to respond at turn 4 +2025-12-27 10:17:23,198 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:24:45,981 - WARNING - User agent failed to respond at turn 4 +2025-12-27 10:41:24,671 - INFO - Profile 11/30 +2025-12-27 11:38:47,631 - INFO - Profile 12/30 +2025-12-27 12:19:31,005 - WARNING - User agent failed to respond at turn 4 +2025-12-27 12:31:03,630 - INFO - Profile 13/30 +2025-12-27 12:54:50,025 - WARNING - User agent failed to respond at turn 6 +2025-12-27 12:58:36,759 - WARNING - User agent failed to respond at turn 3 +2025-12-27 13:02:04,376 - WARNING - User agent failed to respond at turn 5 +2025-12-27 13:14:20,302 - INFO - Profile 14/30 +2025-12-27 13:31:29,268 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:34:19,996 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:02:04,997 - INFO - Profile 15/30 +2025-12-27 14:21:27,253 - WARNING - User agent failed to respond at turn 4 +2025-12-27 14:41:02,004 - INFO - Profile 16/30 +2025-12-27 15:11:47,767 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:15:59,218 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:23:18,670 - INFO - Profile 17/30 +2025-12-27 15:26:29,631 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:41:28,028 - WARNING - User agent failed to respond at turn 5 +2025-12-27 16:16:11,075 - WARNING - User agent failed to respond at turn 2 +2025-12-27 16:25:49,209 - INFO - Profile 18/30 +2025-12-27 17:14:30,971 - INFO - Profile 19/30 +2025-12-27 17:23:08,123 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:20:12,384 - INFO - Profile 20/30 +2025-12-27 18:24:15,177 - WARNING - User agent failed to respond at turn 3 +2025-12-27 18:31:50,579 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:39:24,278 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:51:42,364 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:59:25,266 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:14:22,797 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:34:17,612 - INFO - Profile 21/30 +2025-12-27 19:56:12,348 - WARNING - User agent failed to respond at turn 2 +2025-12-27 20:03:59,021 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:10:11,631 - WARNING - User agent failed to respond at turn 6 +2025-12-27 20:26:56,963 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:30:31,496 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:34:38,513 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:53:09,810 - INFO - Profile 22/30 +2025-12-27 21:48:05,766 - INFO - Profile 23/30 +2025-12-27 22:28:29,304 - WARNING - User agent failed to respond at turn 4 +2025-12-27 22:59:13,258 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:59:13,261 - INFO - Profile 24/30 +2025-12-27 23:06:39,301 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:19:24,621 - WARNING - User agent failed to respond at turn 2 +2025-12-27 23:38:18,263 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:55:20,391 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:55:20,393 - INFO - Profile 25/30 +2025-12-28 00:02:06,164 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:08:19,823 - WARNING - User agent failed to respond at turn 2 +2025-12-28 00:23:05,450 - WARNING - User agent failed to respond at turn 2 +2025-12-28 00:52:08,278 - INFO - Profile 26/30 +2025-12-28 00:55:08,573 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:00:58,753 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:03:09,618 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:19:09,162 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:35:04,963 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:37:11,449 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:40:14,204 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:50:50,908 - INFO - Profile 27/30 +2025-12-28 01:53:17,203 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:11:07,167 - WARNING - User agent failed to respond at turn 4 +2025-12-28 02:24:30,855 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:48:24,373 - WARNING - User agent failed to respond at turn 4 +2025-12-28 02:48:24,376 - INFO - Profile 28/30 +2025-12-28 03:09:58,744 - WARNING - User agent failed to respond at turn 4 +2025-12-28 03:18:04,465 - WARNING - User agent failed to respond at turn 3 +2025-12-28 03:31:51,389 - WARNING - User agent failed to respond at turn 7 +2025-12-28 03:44:54,395 - INFO - Profile 29/30 +2025-12-28 04:17:56,139 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:51:52,370 - INFO - Profile 30/30 +2025-12-28 05:22:33,090 - WARNING - User agent failed to respond at turn 4 +2025-12-28 05:25:11,094 - WARNING - User agent failed to respond at turn 2 +2025-12-28 05:35:41,114 - INFO - Report saved to ../results/rag_vector_20251227_015842/20251227_015904/report.md diff --git a/collaborativeagents/slurm/logs/reflection_14360424.err b/collaborativeagents/slurm/logs/reflection_14360424.err new file mode 100644 index 0000000..551acf5 --- /dev/null +++ b/collaborativeagents/slurm/logs/reflection_14360424.err @@ -0,0 +1,101 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 02:02:07,264 - INFO - Loaded dataset: mmlu +2025-12-27 02:02:07,264 - INFO - Loaded dataset: aime +2025-12-27 02:02:07,264 - INFO - Loaded dataset: math-hard +2025-12-27 02:02:07,264 - INFO - Loaded dataset: humaneval +2025-12-27 02:02:07,278 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 02:02:07,278 - INFO - Running method: reflection +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.51s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.24s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.47s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.57s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +2025-12-27 02:02:32,181 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:04<00:33, 4.24s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:09<00:32, 4.71s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:14<00:29, 4.91s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:19<00:25, 5.00s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:25<00:21, 5.32s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:30<00:15, 5.23s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [00:35<00:10, 5.14s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [00:38<00:04, 4.62s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:40<00:00, 3.75s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:40<00:00, 4.54s/it] +2025-12-27 02:39:58,321 - WARNING - User agent failed to respond at turn 4 +2025-12-27 02:51:18,723 - INFO - Profile 2/30 +2025-12-27 03:42:18,671 - INFO - Profile 3/30 +2025-12-27 04:06:36,514 - WARNING - User agent failed to respond at turn 5 +2025-12-27 04:22:16,874 - WARNING - User agent failed to respond at turn 9 +2025-12-27 04:25:25,486 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:34:09,765 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:47:50,266 - INFO - Profile 4/30 +2025-12-27 05:08:18,870 - WARNING - User agent failed to respond at turn 2 +2025-12-27 06:04:25,163 - INFO - Profile 5/30 +2025-12-27 06:46:34,782 - INFO - Profile 6/30 +2025-12-27 07:37:47,157 - INFO - Profile 7/30 +2025-12-27 07:41:44,319 - WARNING - User agent failed to respond at turn 5 +2025-12-27 07:44:57,141 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:31:18,391 - WARNING - User agent failed to respond at turn 4 +2025-12-27 08:53:24,527 - INFO - Profile 8/30 +2025-12-27 09:02:17,811 - WARNING - User agent failed to respond at turn 2 +2025-12-27 09:10:00,697 - WARNING - User agent failed to respond at turn 2 +2025-12-27 10:06:01,427 - INFO - Profile 9/30 +2025-12-27 10:10:10,794 - WARNING - User agent failed to respond at turn 2 +2025-12-27 10:54:26,451 - INFO - Profile 10/30 +2025-12-27 11:45:46,800 - INFO - Profile 11/30 +2025-12-27 12:11:54,366 - WARNING - User agent failed to respond at turn 4 +2025-12-27 12:37:49,938 - WARNING - User agent failed to respond at turn 5 +2025-12-27 12:53:20,643 - INFO - Profile 12/30 +2025-12-27 13:21:19,712 - WARNING - User agent failed to respond at turn 3 +2025-12-27 13:52:22,200 - INFO - Profile 13/30 +2025-12-27 14:32:11,397 - WARNING - User agent failed to respond at turn 1 +2025-12-27 14:40:54,805 - INFO - Profile 14/30 +2025-12-27 15:32:12,165 - INFO - Profile 15/30 +2025-12-27 16:24:14,011 - INFO - Profile 16/30 +2025-12-27 17:10:39,805 - INFO - Profile 17/30 +2025-12-27 17:50:48,532 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:17:20,986 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:17:39,723 - INFO - Profile 18/30 +2025-12-27 18:18:10,896 - WARNING - User agent failed to respond at turn 0 +2025-12-27 18:50:45,817 - WARNING - User agent failed to respond at turn 2 +2025-12-27 19:18:30,976 - INFO - Profile 19/30 +2025-12-27 20:21:34,652 - INFO - Profile 20/30 +2025-12-27 20:26:57,365 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:30:35,756 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:42:35,308 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:48:30,267 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:51:42,151 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:55:16,916 - WARNING - User agent failed to respond at turn 4 +2025-12-27 21:17:50,813 - WARNING - User agent failed to respond at turn 4 +2025-12-27 21:26:06,307 - WARNING - User agent failed to respond at turn 7 +2025-12-27 21:50:43,671 - INFO - Profile 21/30 +2025-12-27 22:35:18,540 - WARNING - User agent failed to respond at turn 5 +2025-12-27 22:53:01,391 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:13:29,157 - INFO - Profile 22/30 +2025-12-27 23:51:06,223 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:17:23,488 - INFO - Profile 23/30 +2025-12-28 01:19:07,558 - WARNING - User agent failed to respond at turn 10 +2025-12-28 01:37:49,690 - WARNING - User agent failed to respond at turn 6 +2025-12-28 01:38:08,367 - INFO - Profile 24/30 +2025-12-28 02:41:38,285 - INFO - Profile 25/30 +2025-12-28 03:52:11,789 - INFO - Profile 26/30 +2025-12-28 04:23:39,557 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:34:09,940 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:46:57,468 - WARNING - User agent failed to respond at turn 3 +2025-12-28 05:05:43,353 - INFO - Profile 27/30 +2025-12-28 05:38:16,855 - WARNING - User agent failed to respond at turn 2 +2025-12-28 06:17:59,240 - INFO - Profile 28/30 +2025-12-28 06:23:46,429 - WARNING - User agent failed to respond at turn 3 +2025-12-28 06:40:27,991 - WARNING - User agent failed to respond at turn 4 +2025-12-28 07:06:02,435 - WARNING - User agent failed to respond at turn 3 +2025-12-28 07:16:42,790 - WARNING - User agent failed to respond at turn 4 +2025-12-28 07:34:31,553 - INFO - Profile 29/30 +2025-12-28 08:28:07,646 - WARNING - User agent failed to respond at turn 4 +2025-12-28 08:53:34,126 - INFO - Profile 30/30 +2025-12-28 09:32:37,813 - WARNING - User agent failed to respond at turn 7 +2025-12-28 09:40:54,513 - INFO - Report saved to ../results/reflection_20251227_020146/20251227_020207/report.md diff --git a/collaborativeagents/slurm/logs/reflection_grpo_14360425.err b/collaborativeagents/slurm/logs/reflection_grpo_14360425.err new file mode 100644 index 0000000..3b427bf --- /dev/null +++ b/collaborativeagents/slurm/logs/reflection_grpo_14360425.err @@ -0,0 +1,119 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 02:31:09,224 - INFO - Loaded dataset: mmlu +2025-12-27 02:31:09,224 - INFO - Loaded dataset: aime +2025-12-27 02:31:09,224 - INFO - Loaded dataset: math-hard +2025-12-27 02:31:09,224 - INFO - Loaded dataset: humaneval +2025-12-27 02:31:09,299 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 02:31:09,299 - INFO - Running method: reflection_grpo +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.49s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.14s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.13s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.07s/it] +2025-12-27 02:31:33,591 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:03<00:31, 3.95s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:08<00:31, 4.48s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:13<00:27, 4.63s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:18<00:23, 4.75s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:23<00:19, 4.93s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:29<00:15, 5.04s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [00:33<00:09, 4.96s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [00:37<00:04, 4.46s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 3.63s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 4.34s/it] +2025-12-27 03:43:56,265 - WARNING - User agent failed to respond at turn 4 +2025-12-27 03:44:19,363 - INFO - Profile 2/30 +2025-12-27 04:41:18,679 - INFO - Profile 3/30 +2025-12-27 04:50:08,015 - WARNING - User agent failed to respond at turn 6 +2025-12-27 05:37:09,400 - WARNING - User agent failed to respond at turn 4 +2025-12-27 05:39:22,155 - INFO - Profile 4/30 +2025-12-27 05:51:40,082 - WARNING - User agent failed to respond at turn 4 +2025-12-27 06:30:54,910 - WARNING - User agent failed to respond at turn 4 +2025-12-27 06:55:12,778 - INFO - Profile 5/30 +2025-12-27 07:48:39,008 - INFO - Profile 6/30 +2025-12-27 08:01:42,219 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:23:42,492 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:54:04,212 - INFO - Profile 7/30 +2025-12-27 08:58:13,539 - WARNING - User agent failed to respond at turn 3 +2025-12-27 09:24:36,991 - WARNING - User agent failed to respond at turn 7 +2025-12-27 10:01:43,345 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:04:41,897 - WARNING - User agent failed to respond at turn 2 +2025-12-27 10:20:11,751 - INFO - Profile 8/30 +2025-12-27 11:08:02,876 - WARNING - User agent failed to respond at turn 4 +2025-12-27 11:20:28,004 - WARNING - User agent failed to respond at turn 5 +2025-12-27 11:46:14,996 - WARNING - User agent failed to respond at turn 4 +2025-12-27 11:46:33,648 - INFO - Profile 9/30 +2025-12-27 12:22:26,369 - WARNING - User agent failed to respond at turn 9 +2025-12-27 12:56:13,166 - INFO - Profile 10/30 +2025-12-27 13:02:01,791 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:24:51,498 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:16:50,083 - INFO - Profile 11/30 +2025-12-27 14:27:09,697 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:40:22,936 - INFO - Profile 12/30 +2025-12-27 15:52:57,164 - WARNING - User agent failed to respond at turn 5 +2025-12-27 16:28:17,345 - WARNING - User agent failed to respond at turn 4 +2025-12-27 16:50:21,596 - INFO - Profile 13/30 +2025-12-27 17:41:49,444 - WARNING - User agent failed to respond at turn 4 +2025-12-27 17:50:43,295 - INFO - Profile 14/30 +2025-12-27 18:08:38,210 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:44:39,617 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:47:39,503 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:00:23,116 - INFO - Profile 15/30 +2025-12-27 19:12:53,841 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:03:36,023 - INFO - Profile 16/30 +2025-12-27 20:50:11,725 - INFO - Profile 17/30 +2025-12-27 20:54:36,277 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:18:15,804 - INFO - Profile 18/30 +2025-12-27 22:40:24,135 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:04:20,252 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:23:13,204 - INFO - Profile 19/30 +2025-12-28 00:30:41,183 - INFO - Profile 20/30 +2025-12-28 01:13:04,372 - WARNING - User agent failed to respond at turn 6 +2025-12-28 01:21:59,883 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:47:59,918 - WARNING - User agent failed to respond at turn 7 +2025-12-28 01:53:24,077 - WARNING - User agent failed to respond at turn 5 +2025-12-28 02:13:56,170 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:14:14,770 - INFO - Profile 21/30 +2025-12-28 03:20:09,605 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:10:58,912 - INFO - Profile 22/30 +2025-12-28 05:16:04,670 - WARNING - User agent failed to respond at turn 4 +2025-12-28 05:29:32,044 - INFO - Profile 23/30 +2025-12-28 05:46:53,577 - WARNING - User agent failed to respond at turn 6 +2025-12-28 05:57:05,360 - WARNING - User agent failed to respond at turn 6 +2025-12-28 06:14:11,895 - WARNING - User agent failed to respond at turn 5 +2025-12-28 06:21:21,665 - WARNING - User agent failed to respond at turn 3 +2025-12-28 06:43:49,754 - INFO - Profile 24/30 +2025-12-28 06:56:35,737 - WARNING - User agent failed to respond at turn 3 +2025-12-28 07:54:52,613 - INFO - Profile 25/30 +2025-12-28 08:24:24,212 - WARNING - User agent failed to respond at turn 2 +2025-12-28 09:01:32,435 - INFO - Profile 26/30 +2025-12-28 09:24:20,607 - WARNING - User agent failed to respond at turn 4 +2025-12-28 09:28:45,402 - WARNING - User agent failed to respond at turn 3 +2025-12-28 09:31:07,307 - WARNING - User agent failed to respond at turn 2 +2025-12-28 09:37:47,214 - WARNING - User agent failed to respond at turn 2 +2025-12-28 09:49:55,833 - WARNING - User agent failed to respond at turn 3 +2025-12-28 09:54:03,278 - WARNING - User agent failed to respond at turn 2 +2025-12-28 10:13:11,944 - INFO - Profile 27/30 +2025-12-28 10:34:58,991 - WARNING - User agent failed to respond at turn 5 +2025-12-28 10:42:04,222 - WARNING - User agent failed to respond at turn 4 +2025-12-28 10:49:59,056 - WARNING - User agent failed to respond at turn 3 +2025-12-28 11:22:55,596 - WARNING - User agent failed to respond at turn 2 +2025-12-28 11:36:59,263 - INFO - Profile 28/30 +2025-12-28 11:49:22,297 - WARNING - User agent failed to respond at turn 6 +2025-12-28 11:52:54,358 - WARNING - User agent failed to respond at turn 3 +2025-12-28 12:04:04,093 - WARNING - User agent failed to respond at turn 4 +2025-12-28 12:11:08,251 - WARNING - User agent failed to respond at turn 3 +2025-12-28 12:17:19,369 - WARNING - User agent failed to respond at turn 6 +2025-12-28 12:20:37,784 - WARNING - User agent failed to respond at turn 4 +2025-12-28 12:33:13,220 - WARNING - User agent failed to respond at turn 4 +2025-12-28 12:41:36,621 - WARNING - User agent failed to respond at turn 5 +2025-12-28 12:44:34,822 - WARNING - User agent failed to respond at turn 3 +2025-12-28 12:50:56,698 - WARNING - User agent failed to respond at turn 6 +2025-12-28 12:56:12,269 - INFO - Profile 29/30 +2025-12-28 14:22:05,011 - INFO - Profile 30/30 +2025-12-28 15:13:10,367 - INFO - Report saved to ../results/reflection_grpo_20251227_023047/20251227_023109/report.md diff --git a/collaborativeagents/slurm/logs/run_collab_baselines_14355966.err b/collaborativeagents/slurm/logs/run_collab_baselines_14355966.err new file mode 100644 index 0000000..6d3191e --- /dev/null +++ b/collaborativeagents/slurm/logs/run_collab_baselines_14355966.err @@ -0,0 +1,14466 @@ +2025-12-25 10:37:47,003 - INFO - Loaded dataset: math-500 +2025-12-25 10:37:47,143 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 10:37:47,144 - INFO - Running method: contextual +2025-12-25 10:37:51,592 - INFO - Profile 1/20 +2025-12-25 10:37:53,048 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,062 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,063 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,066 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,067 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,069 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,070 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,073 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,074 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,076 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,077 - INFO - Profile 2/20 +2025-12-25 10:37:53,077 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,080 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,081 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,083 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,084 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,087 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,088 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,090 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,091 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,094 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,094 - INFO - Profile 3/20 +2025-12-25 10:37:53,094 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,097 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,098 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,101 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,101 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,104 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,105 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,107 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,108 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,111 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,111 - INFO - Profile 4/20 +2025-12-25 10:37:53,112 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,114 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,115 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,118 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,118 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,121 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,122 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,124 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,125 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,128 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,128 - INFO - Profile 5/20 +2025-12-25 10:37:53,129 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,131 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,132 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,135 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,136 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,138 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,139 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,142 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,142 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,145 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,145 - INFO - Profile 6/20 +2025-12-25 10:37:53,146 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,149 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,149 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,152 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,153 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,156 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,156 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,159 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,160 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,162 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,163 - INFO - Profile 7/20 +2025-12-25 10:37:53,163 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,166 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,167 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,169 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,170 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,173 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,174 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,176 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,177 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,180 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,180 - INFO - Profile 8/20 +2025-12-25 10:37:53,181 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,184 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,184 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,187 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,188 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,190 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,191 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,194 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,195 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,198 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,198 - INFO - Profile 9/20 +2025-12-25 10:37:53,198 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,201 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,202 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,205 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,206 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,208 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,209 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,212 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,213 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,215 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,215 - INFO - Profile 10/20 +2025-12-25 10:37:53,216 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,219 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,220 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,222 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,223 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,226 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,227 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,230 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,231 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,233 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,233 - INFO - Profile 11/20 +2025-12-25 10:37:53,234 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,237 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,238 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,240 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,241 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,244 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,245 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,247 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,248 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,251 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,251 - INFO - Profile 12/20 +2025-12-25 10:37:53,252 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,255 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,255 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,258 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,259 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,262 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,263 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,265 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,266 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,269 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,269 - INFO - Profile 13/20 +2025-12-25 10:37:53,270 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,273 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,273 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,276 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,277 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,280 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,281 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,283 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,284 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,287 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,287 - INFO - Profile 14/20 +2025-12-25 10:37:53,288 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,291 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,291 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,294 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,295 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,298 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,299 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,301 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,302 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,305 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,305 - INFO - Profile 15/20 +2025-12-25 10:37:53,306 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,309 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,310 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,312 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,313 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,316 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,317 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,320 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,321 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,323 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,323 - INFO - Profile 16/20 +2025-12-25 10:37:53,324 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,327 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,328 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,331 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,332 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,334 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,335 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,338 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,339 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,341 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,342 - INFO - Profile 17/20 +2025-12-25 10:37:53,343 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,345 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,346 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,349 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,350 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,352 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,354 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,356 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,357 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,360 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,360 - INFO - Profile 18/20 +2025-12-25 10:37:53,361 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,364 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,365 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,367 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,368 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,371 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,372 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,375 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,376 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,378 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,379 - INFO - Profile 19/20 +2025-12-25 10:37:53,380 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,382 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,383 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,386 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,387 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,390 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,391 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,393 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,394 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,397 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,397 - INFO - Profile 20/20 +2025-12-25 10:37:53,398 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,401 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,402 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,405 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,406 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,408 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,409 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,412 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,413 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,416 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,422 - INFO - Running method: reflection +2025-12-25 10:37:53,427 - INFO - Profile 1/20 +2025-12-25 10:37:53,428 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,431 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,432 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,435 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,435 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,438 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,439 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,441 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,442 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,445 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,445 - INFO - Profile 2/20 +2025-12-25 10:37:53,445 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,448 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,449 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,451 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,452 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,455 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,455 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,458 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,459 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,461 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,462 - INFO - Profile 3/20 +2025-12-25 10:37:53,462 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,465 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,466 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,468 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,469 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,472 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,472 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,475 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,476 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,478 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,478 - INFO - Profile 4/20 +2025-12-25 10:37:53,479 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,482 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,482 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,485 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,486 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,488 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,489 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,492 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,493 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,495 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,495 - INFO - Profile 5/20 +2025-12-25 10:37:53,496 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,499 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,499 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,502 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,503 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,505 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,506 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,509 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,509 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,513 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,513 - INFO - Profile 6/20 +2025-12-25 10:37:53,514 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,517 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,517 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,520 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,521 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,523 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,524 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,527 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,527 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,530 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,530 - INFO - Profile 7/20 +2025-12-25 10:37:53,531 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,534 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,534 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,537 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,538 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,540 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,541 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,544 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,544 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,547 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,547 - INFO - Profile 8/20 +2025-12-25 10:37:53,548 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,550 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,551 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,554 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,554 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,557 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,558 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,560 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,561 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,564 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,564 - INFO - Profile 9/20 +2025-12-25 10:37:53,565 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,567 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,568 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,571 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,571 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,574 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,575 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,577 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,578 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,581 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,581 - INFO - Profile 10/20 +2025-12-25 10:37:53,581 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,584 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,585 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,587 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,588 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,591 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,591 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,594 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,595 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,597 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,598 - INFO - Profile 11/20 +2025-12-25 10:37:53,598 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,601 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,602 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,604 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,605 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,608 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,608 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,611 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,612 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,614 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,614 - INFO - Profile 12/20 +2025-12-25 10:37:53,615 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,618 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,618 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,621 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,622 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,624 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,625 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,628 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,628 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,631 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,631 - INFO - Profile 13/20 +2025-12-25 10:37:53,632 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,635 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,635 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,638 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,639 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,641 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,642 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,645 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,645 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,648 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,648 - INFO - Profile 14/20 +2025-12-25 10:37:53,649 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,651 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,652 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,655 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,655 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,658 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,659 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,661 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,662 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,665 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,665 - INFO - Profile 15/20 +2025-12-25 10:37:53,666 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,668 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,669 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,672 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,672 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,675 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,676 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,678 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,679 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,682 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,682 - INFO - Profile 16/20 +2025-12-25 10:37:53,682 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,685 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,686 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,688 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,689 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,692 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,692 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,695 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,696 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,698 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,699 - INFO - Profile 17/20 +2025-12-25 10:37:53,699 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,702 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,703 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,705 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,706 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,709 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,709 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,712 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,713 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,715 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,715 - INFO - Profile 18/20 +2025-12-25 10:37:53,716 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,719 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,719 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,722 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,723 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,725 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,726 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,729 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,729 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,732 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,732 - INFO - Profile 19/20 +2025-12-25 10:37:53,733 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,735 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,736 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,739 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,739 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,742 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,743 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,746 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,746 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,749 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,749 - INFO - Profile 20/20 +2025-12-25 10:37:53,750 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,752 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,753 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,756 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,756 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,759 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,760 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,762 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,763 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,766 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,785 - INFO - Running method: reflection_grpo +2025-12-25 10:37:53,791 - INFO - Profile 1/20 +2025-12-25 10:37:53,791 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,795 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,795 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,798 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,799 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,801 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,802 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,805 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,805 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,808 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,808 - INFO - Profile 2/20 +2025-12-25 10:37:53,809 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,812 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,812 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,815 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,816 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,818 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,819 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,822 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,823 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,825 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,825 - INFO - Profile 3/20 +2025-12-25 10:37:53,826 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,829 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,829 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,832 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,833 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,835 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,836 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,839 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,840 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,842 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,842 - INFO - Profile 4/20 +2025-12-25 10:37:53,843 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,846 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,846 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,849 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,850 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,853 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,853 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,856 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,857 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,859 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,859 - INFO - Profile 5/20 +2025-12-25 10:37:53,860 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,863 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,864 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,866 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,867 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,870 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,870 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,873 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,874 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,876 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,876 - INFO - Profile 6/20 +2025-12-25 10:37:53,877 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,880 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,881 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,883 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,884 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,887 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,887 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,890 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,891 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,893 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,893 - INFO - Profile 7/20 +2025-12-25 10:37:53,894 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,897 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,897 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,900 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,901 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,903 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,904 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,907 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,908 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,910 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,910 - INFO - Profile 8/20 +2025-12-25 10:37:53,911 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,914 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,914 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,917 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,918 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,920 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,921 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,924 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,925 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,927 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,927 - INFO - Profile 9/20 +2025-12-25 10:37:53,928 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,931 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,931 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,934 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,935 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,937 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,938 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,941 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,942 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,944 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,944 - INFO - Profile 10/20 +2025-12-25 10:37:53,945 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,948 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,949 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,951 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,952 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,955 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,955 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,958 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,959 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,961 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,961 - INFO - Profile 11/20 +2025-12-25 10:37:53,962 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,965 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,966 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,968 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,969 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,972 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,972 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,975 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,976 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,978 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,979 - INFO - Profile 12/20 +2025-12-25 10:37:53,979 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,982 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,983 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,985 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,986 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,989 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,989 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,992 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,993 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,995 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,995 - INFO - Profile 13/20 +2025-12-25 10:37:53,996 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,999 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,000 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,002 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,003 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,006 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,006 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,009 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,010 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,012 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,012 - INFO - Profile 14/20 +2025-12-25 10:37:54,013 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,016 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,017 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,019 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,020 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,023 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,023 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,026 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,027 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,029 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,030 - INFO - Profile 15/20 +2025-12-25 10:37:54,030 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,033 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,034 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,036 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,037 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,040 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,040 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,043 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,044 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,047 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,047 - INFO - Profile 16/20 +2025-12-25 10:37:54,047 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,050 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,051 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,053 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,054 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,057 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,057 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,060 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,061 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,064 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,064 - INFO - Profile 17/20 +2025-12-25 10:37:54,064 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,067 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,068 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,070 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,071 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,074 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,074 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,077 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,078 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,080 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,081 - INFO - Profile 18/20 +2025-12-25 10:37:54,081 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,084 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,085 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,087 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,088 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,091 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,091 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,094 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,095 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,097 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,098 - INFO - Profile 19/20 +2025-12-25 10:37:54,098 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,101 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,102 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,104 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,105 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,108 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,108 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,111 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,112 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,114 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,115 - INFO - Profile 20/20 +2025-12-25 10:37:54,115 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,118 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,119 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,121 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,122 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,125 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,125 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,128 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,129 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,131 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,141 - INFO - Report saved to ../results/collab_baselines_20251225_103724/20251225_103746/report.md diff --git a/collaborativeagents/slurm/logs/run_collab_baselines_14355975.err b/collaborativeagents/slurm/logs/run_collab_baselines_14355975.err new file mode 100644 index 0000000..c82b955 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_collab_baselines_14355975.err @@ -0,0 +1,70 @@ +2025-12-25 10:52:43,142 - INFO - Loaded dataset: math-500 +2025-12-25 10:52:43,385 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 10:52:43,386 - INFO - Running method: contextual +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.47s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.43s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.59s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +2025-12-25 10:53:08,251 - INFO - Profile 1/20 +2025-12-25 10:54:31,245 - INFO - Profile 2/20 +2025-12-25 10:55:49,021 - INFO - Profile 3/20 +2025-12-25 10:57:07,721 - INFO - Profile 4/20 +2025-12-25 10:57:46,873 - INFO - Profile 5/20 +2025-12-25 10:58:26,077 - INFO - Profile 6/20 +2025-12-25 10:59:05,298 - INFO - Profile 7/20 +2025-12-25 10:59:44,507 - INFO - Profile 8/20 +2025-12-25 11:00:23,700 - INFO - Profile 9/20 +2025-12-25 11:01:02,919 - INFO - Profile 10/20 +2025-12-25 11:01:42,131 - INFO - Profile 11/20 +2025-12-25 11:02:57,684 - INFO - Profile 12/20 +2025-12-25 11:03:56,775 - INFO - Profile 13/20 +2025-12-25 11:05:03,753 - INFO - Profile 14/20 +2025-12-25 11:05:51,697 - INFO - Profile 15/20 +2025-12-25 11:06:52,101 - INFO - Profile 16/20 +2025-12-25 11:07:27,746 - INFO - Profile 17/20 +2025-12-25 11:08:54,497 - INFO - Profile 18/20 +2025-12-25 11:10:40,288 - INFO - Profile 19/20 +2025-12-25 11:12:07,984 - INFO - Profile 20/20 +2025-12-25 11:12:44,173 - INFO - Running method: reflection +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.00s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:20<00:06, 6.78s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.87s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.49s/it] +2025-12-25 11:13:06,758 - INFO - Profile 1/20 +2025-12-25 11:14:12,362 - INFO - Profile 2/20 +2025-12-25 11:15:35,445 - INFO - Profile 3/20 +2025-12-25 11:16:54,467 - INFO - Profile 4/20 +2025-12-25 11:18:12,199 - INFO - Profile 5/20 +2025-12-25 11:19:42,312 - INFO - Profile 6/20 +2025-12-25 11:21:06,922 - INFO - Profile 7/20 +2025-12-25 11:22:54,996 - INFO - Profile 8/20 +2025-12-25 11:24:08,753 - INFO - Profile 9/20 +2025-12-25 11:25:19,792 - INFO - Profile 10/20 +2025-12-25 11:26:42,922 - INFO - Profile 11/20 +2025-12-25 11:28:16,287 - INFO - Profile 12/20 +2025-12-25 11:29:37,361 - INFO - Profile 13/20 +2025-12-25 11:30:51,174 - INFO - Profile 14/20 +2025-12-25 11:32:25,617 - INFO - Profile 15/20 +2025-12-25 11:33:26,202 - INFO - Profile 16/20 +2025-12-25 11:34:28,137 - INFO - Profile 17/20 +2025-12-25 11:35:45,693 - INFO - Profile 18/20 +2025-12-25 11:37:00,624 - INFO - Profile 19/20 +2025-12-25 11:38:12,427 - INFO - Profile 20/20 +2025-12-25 11:39:29,675 - INFO - Running method: reflection_grpo +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.18s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.71s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.35s/it] +2025-12-25 11:39:51,748 - INFO - Profile 1/20 +2025-12-25 11:41:06,412 - INFO - Profile 2/20 +2025-12-25 11:42:24,983 - INFO - Profile 3/20 +2025-12-25 11:43:42,715 - INFO - Profile 4/20 +2025-12-25 11:45:21,420 - INFO - Profile 5/20 +2025-12-25 11:46:41,875 - INFO - Profile 6/20 +2025-12-25 11:48:09,967 - INFO - Profile 7/20 +2025-12-25 11:49:25,792 - INFO - Profile 8/20 +2025-12-25 11:50:50,009 - INFO - Profile 9/20 +2025-12-25 11:52:26,201 - INFO - Profile 10/20 +2025-12-25 11:53:51,842 - INFO - Profile 11/20 +2025-12-25 11:55:20,284 - INFO - Profile 12/20 +2025-12-25 11:56:36,649 - INFO - Profile 13/20 +2025-12-25 11:58:06,896 - INFO - Profile 14/20 +2025-12-25 11:59:31,014 - INFO - Profile 15/20 +2025-12-25 12:00:48,734 - INFO - Profile 16/20 +2025-12-25 12:02:23,732 - INFO - Profile 17/20 +2025-12-25 12:03:20,783 - INFO - Profile 18/20 +2025-12-25 12:04:44,524 - INFO - Profile 19/20 +2025-12-25 12:06:08,471 - INFO - Profile 20/20 +2025-12-25 12:07:36,561 - INFO - Report saved to ../results/collab_baselines_20251225_105235/20251225_105243/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355787.err b/collaborativeagents/slurm/logs/run_expts_a100_14355787.err new file mode 100644 index 0000000..4845249 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355787.err @@ -0,0 +1,15 @@ +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 31, in <module> + from adapters.personalized_llm_adapter import PersonalizedLLMAdapter, create_baseline_adapter + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/__init__.py", line 3, in <module> + from .personalized_llm_adapter import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 21, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/__init__.py", line 5, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 35, in <module> + from personalization.models.reranker.qwen3_reranker import Qwen3Reranker + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/reranker/qwen3_reranker.py", line 26 + self.model = AutoModelForCausalLM.from_pretrained( + ^ +IndentationError: expected an indented block after 'else' statement on line 24 diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355797.err b/collaborativeagents/slurm/logs/run_expts_a100_14355797.err new file mode 100644 index 0000000..28ce959 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355797.err @@ -0,0 +1,14 @@ +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 31, in <module> + from adapters.personalized_llm_adapter import PersonalizedLLMAdapter, create_baseline_adapter + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/__init__.py", line 3, in <module> + from .personalized_llm_adapter import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 21, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/__init__.py", line 5, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 38, in <module> + from personalization.user_model.features import ItemProjection + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/user_model/features.py", line 3, in <module> + from sklearn.decomposition import PCA +ModuleNotFoundError: No module named 'sklearn' diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355807.err b/collaborativeagents/slurm/logs/run_expts_a100_14355807.err new file mode 100644 index 0000000..33fa583 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355807.err @@ -0,0 +1,10 @@ +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 484, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 471, in main + runner = ExperimentRunner(config) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 99, in __init__ + self.conflict_generator = ConflictScenarioGenerator() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: ConflictScenarioGenerator.__init__() missing 1 required positional argument: 'profile' diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355816.err b/collaborativeagents/slurm/logs/run_expts_a100_14355816.err new file mode 100644 index 0000000..f147ab6 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355816.err @@ -0,0 +1,22 @@ +2025-12-25 07:53:36,898 - INFO - Loaded dataset: math-500 +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 484, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 471, in main + runner = ExperimentRunner(config) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 111, in __init__ + self.profiles = self._load_profiles() + ^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 117, in _load_profiles + profiles = json.load(f) + ^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/json/__init__.py", line 293, in load + return loads(fp.read(), + ^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/json/__init__.py", line 346, in loads + return _default_decoder.decode(s) + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/json/decoder.py", line 340, in decode + raise JSONDecodeError("Extra data", s, end) +json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 10782) diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355842.err b/collaborativeagents/slurm/logs/run_expts_a100_14355842.err new file mode 100644 index 0000000..0144ee9 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355842.err @@ -0,0 +1,23 @@ +2025-12-25 07:55:08,518 - INFO - Loaded dataset: math-500 +2025-12-25 07:55:08,527 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 07:55:08,528 - INFO - Running method: rag_vector +2025-12-25 07:55:08,528 - INFO - Profile 1/2 +
Generating test split: 0%| | 0/500 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 500/500 [00:00<00:00, 9910.65 examples/s] +2025-12-25 07:55:12,047 - ERROR - Error in session: No module named 'json_repair' +2025-12-25 07:55:12,050 - ERROR - Error in session: No module named 'json_repair' +2025-12-25 07:55:12,050 - INFO - Profile 2/2 +2025-12-25 07:55:12,053 - ERROR - Error in session: No module named 'json_repair' +2025-12-25 07:55:12,056 - ERROR - Error in session: No module named 'json_repair' +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 488, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 476, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 292, in run_all + analysis = self._analyze_results(all_results) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 365, in _analyze_results + best = max(values, key=values.get) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ValueError: max() arg is an empty sequence diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355851.err b/collaborativeagents/slurm/logs/run_expts_a100_14355851.err new file mode 100644 index 0000000..59bbe1a --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355851.err @@ -0,0 +1,28 @@ +2025-12-25 07:58:42,438 - INFO - Loaded dataset: math-500 +2025-12-25 07:58:42,447 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 07:58:42,448 - INFO - Running method: rag_vector +2025-12-25 07:58:42,448 - INFO - Profile 1/2 +2025-12-25 07:58:47,959 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,959 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,960 - INFO - Profile 2/2 +2025-12-25 07:58:47,960 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,960 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,962 - WARNING - No values for metric task_success_rate, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_user_tokens, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_total_tokens, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_enforcement_count, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_preference_compliance, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric conflict_resolution_accuracy, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric over_personalization_rate, skipping comparison +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 491, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 479, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 299, in run_all + self._generate_report(analysis) + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 414, in _generate_report + best = analysis["comparison"][metric_key]["best_method"] + ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +KeyError: 'task_success_rate' diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355856.err b/collaborativeagents/slurm/logs/run_expts_a100_14355856.err new file mode 100644 index 0000000..608e91f --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355856.err @@ -0,0 +1,10 @@ +2025-12-25 08:08:52,658 - INFO - Loaded dataset: math-500 +2025-12-25 08:08:52,698 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:08:52,699 - INFO - Running method: rag_vector +2025-12-25 08:08:52,699 - INFO - Profile 1/2 +2025-12-25 08:08:55,383 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,383 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,383 - INFO - Profile 2/2 +2025-12-25 08:08:55,384 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,384 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,390 - INFO - Report saved to ../results/test_a100_20251225_080844/20251225_080852/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355861.err b/collaborativeagents/slurm/logs/run_expts_a100_14355861.err new file mode 100644 index 0000000..fac726b --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355861.err @@ -0,0 +1,14 @@ +2025-12-25 08:12:25,638 - INFO - Loaded dataset: math-500 +2025-12-25 08:12:25,647 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:12:25,648 - INFO - Running method: rag_vector +2025-12-25 08:12:25,648 - INFO - Profile 1/2 +2025-12-25 08:12:28,020 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,062 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,062 - INFO - Profile 2/2 +2025-12-25 08:12:28,107 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,155 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,189 - INFO - Report saved to ../results/test_a100_20251225_081218/20251225_081225/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355863.err b/collaborativeagents/slurm/logs/run_expts_a100_14355863.err new file mode 100644 index 0000000..1a41e92 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355863.err @@ -0,0 +1,11 @@ +2025-12-25 08:15:58,390 - INFO - Loaded dataset: math-500 +2025-12-25 08:15:58,399 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:15:58,399 - INFO - Running method: rag_vector +2025-12-25 08:15:58,399 - INFO - Profile 1/2 +`torch_dtype` is deprecated! Use `dtype` instead! +2025-12-25 08:16:02,559 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:02,854 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:02,874 - INFO - Profile 2/2 +2025-12-25 08:16:03,126 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:03,394 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:03,459 - INFO - Report saved to ../results/test_a100_20251225_081551/20251225_081558/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355865.err b/collaborativeagents/slurm/logs/run_expts_a100_14355865.err new file mode 100644 index 0000000..769bc20 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355865.err @@ -0,0 +1,19 @@ +2025-12-25 08:20:02,816 - INFO - Loaded dataset: math-500 +2025-12-25 08:20:02,951 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:20:02,952 - INFO - Running method: rag_vector +2025-12-25 08:20:02,952 - INFO - Profile 1/2 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:22, 7.63s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:14<00:13, 6.92s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:06, 6.94s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.41s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:21, 5.36s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.40s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:16<00:10, 5.38s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:20<00:05, 5.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 4.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 4.58s/it] +2025-12-25 08:20:51,879 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.19s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.23s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.39s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.53s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.29s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.44s/it] +2025-12-25 08:21:14,611 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +2025-12-25 08:21:14,661 - INFO - Profile 2/2 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.22s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.33s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.08s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.45s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.42s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.17s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.73s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.16s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.39s/it] +2025-12-25 08:21:37,445 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.34s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.28s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.06s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.44s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.43s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.24s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.77s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.17s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.41s/it] +2025-12-25 08:22:00,232 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +2025-12-25 08:22:00,292 - INFO - Report saved to ../results/test_a100_20251225_081954/20251225_082002/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355871.err b/collaborativeagents/slurm/logs/run_expts_a100_14355871.err new file mode 100644 index 0000000..447cacb --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355871.err @@ -0,0 +1,16 @@ +2025-12-25 08:28:55,431 - INFO - Loaded dataset: math-500 +2025-12-25 08:28:55,625 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:28:55,626 - INFO - Running method: rag_vector +2025-12-25 08:28:55,626 - INFO - Profile 1/2 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.41s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.48s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 4.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 5.62s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:09<00:36, 9.15s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:14<00:20, 6.98s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:21<00:13, 6.76s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:25<00:05, 5.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 4.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 5.61s/it] +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.54s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.88s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.22s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.34s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.28s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.39s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.70s/it] +2025-12-25 08:31:49,842 - INFO - Profile 2/2 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.69s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.07s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:07, 3.58s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.44s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.76s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.31s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.69s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.94s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.32s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:10, 5.23s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.33s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.50s/it] +2025-12-25 08:33:27,343 - INFO - Report saved to ../results/test_a100_20251225_082834/20251225_082855/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355878.err b/collaborativeagents/slurm/logs/run_expts_a100_14355878.err new file mode 100644 index 0000000..8634294 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355878.err @@ -0,0 +1,109 @@ +2025-12-25 08:39:20,163 - INFO - Loaded dataset: math-500 +2025-12-25 08:39:20,285 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:39:20,285 - INFO - Running method: vanilla +2025-12-25 08:39:20,285 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:10, 5.18s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.72s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.17s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.66s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.62s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.27s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.36s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.85s/it] +2025-12-25 08:40:03,460 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.11s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.15s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.14s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.57s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.07s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.72s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.21s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.32s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.11s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.66s/it] +2025-12-25 08:40:38,434 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.68s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.98s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.20s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.59s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:09, 4.61s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:04, 4.02s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.90s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.30s/it] +2025-12-25 08:41:11,060 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.94s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.15s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.14s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.57s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.06s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.58s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:08, 4.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.94s/it] +2025-12-25 08:41:42,373 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.36s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.22s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.10s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.54s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.07s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.34s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.91s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:11<00:08, 4.20s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.77s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.11s/it] +2025-12-25 08:42:14,703 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:42:14,780 - INFO - Profile 2/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.99s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.14s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.12s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.54s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:08, 4.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.73s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +2025-12-25 08:42:46,213 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.86s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.18s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.22s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.62s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.10s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:08, 2.18s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.01s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.75s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.53s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.83s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.04s/it] +2025-12-25 08:43:19,397 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.68s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.52s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.67s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.45s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.85s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.92s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.52s/it] +2025-12-25 08:43:48,538 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.34s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.81s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.73s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.94s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.98s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:07, 3.59s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.34s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.89s/it] +2025-12-25 08:44:18,865 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.03s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.63s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.29s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.44s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.09s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.76s/it] +2025-12-25 08:44:48,291 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:44:48,358 - INFO - Profile 3/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.77s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.55s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.70s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.38s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.52s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.60s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.24s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.58s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.90s/it] +2025-12-25 08:45:18,280 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.91s/it]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.09s/it] +2025-12-25 08:45:24,840 - ERROR - Error in session: CUDA out of memory. Tried to allocate 96.00 MiB. GPU 2 has a total capacity of 39.49 GiB of which 90.31 MiB is free. Including non-PyTorch memory, this process has 39.40 GiB memory in use. Of the allocated memory 38.90 GiB is allocated by PyTorch, and 99.47 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.61s/it]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.83s/it] +2025-12-25 08:45:31,127 - ERROR - Error in session: CUDA out of memory. Tried to allocate 96.00 MiB. GPU 1 has a total capacity of 39.49 GiB of which 8.31 MiB is free. Including non-PyTorch memory, this process has 39.48 GiB memory in use. Of the allocated memory 38.89 GiB is allocated by PyTorch, and 181.97 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.57s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.38s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.24s/it] +2025-12-25 08:45:40,072 - ERROR - Error in session: CUDA out of memory. Tried to allocate 96.00 MiB. GPU 1 has a total capacity of 39.49 GiB of which 8.31 MiB is free. Including non-PyTorch memory, this process has 39.48 GiB memory in use. Of the allocated memory 38.89 GiB is allocated by PyTorch, and 181.97 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.61s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.51s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.60s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.65s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:17, 5.73s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.17s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.20s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.19s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.79s/it] +2025-12-25 08:46:14,167 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:46:14,242 - INFO - Profile 4/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.46s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.72s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.74s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.28s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:17, 5.84s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.20s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.31s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.88s/it] +2025-12-25 08:46:48,967 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.93s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.46s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.91s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.99s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.47s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.92s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.41s/it] +2025-12-25 08:47:21,876 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.67s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:09, 4.71s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.36s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.70s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.23s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:16, 5.51s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.85s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.00s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.59s/it] +2025-12-25 08:47:56,567 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.83s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.83s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.44s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.75s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.30s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.56s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.71s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.03s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.64s/it] +2025-12-25 08:48:31,945 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.19s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:09, 4.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.40s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.72s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.27s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.87s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.29s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.83s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.29s/it] +2025-12-25 08:49:05,843 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:49:05,907 - INFO - Profile 5/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:09, 4.59s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.19s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.77s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.49s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.74s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.00s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.42s/it] +2025-12-25 08:49:39,852 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.79s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.78s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.93s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.89s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:08, 2.12s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.37s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.63s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.36s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.65s/it] +2025-12-25 08:50:14,293 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.02s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.70s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.19s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.04s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.66s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.82s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.95s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.03s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.51s/it] +2025-12-25 08:50:48,033 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.00s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.64s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.95s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.46s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.89s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.73s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.68s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.94s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.43s/it] +2025-12-25 08:51:20,931 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.69s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.68s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.03s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.92s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.73s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.92s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.56s/it] +2025-12-25 08:51:54,525 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:51:54,585 - INFO - Profile 6/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.24s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.61s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:15, 5.24s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.23s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.25s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.77s/it] +2025-12-25 08:52:27,892 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.12s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.10s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.64s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.12s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.80s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.74s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.03s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.15s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.59s/it] +2025-12-25 08:53:02,452 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.93s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.93s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:12, 4.17s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.77s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.94s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.33s/it] +2025-12-25 08:53:35,074 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.10s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.82s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.13s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.02s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.67s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.44s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.87s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.33s/it] +2025-12-25 08:54:08,083 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.08s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.04s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 3.00s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.33s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.69s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.77s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.18s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.24s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.68s/it] +2025-12-25 08:54:42,574 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:54:42,637 - INFO - Profile 7/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.87s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.39s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.44s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.75s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.25s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.57s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:09, 4.58s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.43s/it] +2025-12-25 08:55:16,985 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.10s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.86s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.26s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.92s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:15, 3.79s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.50s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.19s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.16s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.85s/it] +2025-12-25 08:55:56,547 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.74s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.80s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.11s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.55s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.24s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.60s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.32s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.34s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 4.18s/it] +2025-12-25 08:56:33,918 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.98s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.82s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.10s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.55s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 3.00s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.24s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.43s/it][2025-12-25T08:56:58.364] error: *** JOB 14355878 ON gpua055 CANCELLED AT 2025-12-25T08:56:58 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355885.err b/collaborativeagents/slurm/logs/run_expts_a100_14355885.err new file mode 100644 index 0000000..ffc2040 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355885.err @@ -0,0 +1,17 @@ +2025-12-25 08:58:07,929 - INFO - Loaded dataset: math-500 +2025-12-25 08:58:08,121 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:58:08,122 - INFO - Running method: vanilla +2025-12-25 08:58:08,122 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.19s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.10s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.46s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:14, 3.68s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.46s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.22s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.30s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.91s/it] +2025-12-25 08:58:47,629 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.44s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.11s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.27s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.65s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:13, 3.45s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:15, 5.12s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.65s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.00s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.10s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.67s/it] +2025-12-25 08:59:23,124 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.95s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.41s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:15, 5.30s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.86s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.12s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.11s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.68s/it] +2025-12-25 08:59:57,630 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.82s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.87s/it][2025-12-25T09:00:23.259] error: *** JOB 14355885 ON gpua055 CANCELLED AT 2025-12-25T09:00:23 DUE to SIGNAL Terminated *** +
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.74s/it]
\ No newline at end of file diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355888.err b/collaborativeagents/slurm/logs/run_expts_a100_14355888.err new file mode 100644 index 0000000..c5abe31 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355888.err @@ -0,0 +1,19 @@ +2025-12-25 09:02:11,022 - INFO - Loaded dataset: math-500 +2025-12-25 09:02:11,141 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:02:11,142 - INFO - Running method: vanilla +2025-12-25 09:02:11,142 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.63s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.64s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.83s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.71s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:10, 5.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.38s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.78s/it] +2025-12-25 09:02:50,489 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.48s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.72s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.85s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.05s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.15s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.63s/it] +2025-12-25 09:03:24,392 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.79s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.62s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.90s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.34s/it] +2025-12-25 09:03:57,453 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.46s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.87s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.79s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.26s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.61s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.80s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.28s/it] +2025-12-25 09:04:31,571 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.83s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.43s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.86s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.40s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.81s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.85s/it][2025-12-25T09:04:49.962] error: *** JOB 14355888 ON gpua055 CANCELLED AT 2025-12-25T09:04:49 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355896.err b/collaborativeagents/slurm/logs/run_expts_a100_14355896.err new file mode 100644 index 0000000..1f92e5d --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355896.err @@ -0,0 +1,12 @@ +2025-12-25 09:10:36,065 - INFO - Loaded dataset: math-500 +2025-12-25 09:10:36,277 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:10:36,278 - INFO - Running method: vanilla +2025-12-25 09:10:36,278 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.40s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.74s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:20<00:06, 6.83s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:07<00:30, 7.52s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:13<00:19, 6.37s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:19<00:13, 6.51s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:24<00:05, 5.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:27<00:00, 4.71s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:27<00:00, 5.48s/it] +2025-12-25 09:11:36,637 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.37s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.27s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.55s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.21s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.58s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.95s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.02s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.48s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.47s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.81s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.97s/it] +2025-12-25 09:12:05,675 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.75s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.40s/it][2025-12-25T09:12:15.828] error: *** JOB 14355896 ON gpua036 CANCELLED AT 2025-12-25T09:12:15 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355901.err b/collaborativeagents/slurm/logs/run_expts_a100_14355901.err new file mode 100644 index 0000000..4ce1958 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355901.err @@ -0,0 +1,47 @@ +2025-12-25 09:12:44,146 - INFO - Loaded dataset: math-500 +2025-12-25 09:12:44,269 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:12:44,270 - INFO - Running method: vanilla +2025-12-25 09:12:44,270 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.16s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.19s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.54s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.97s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.61s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.94s/it] +2025-12-25 09:13:17,230 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 09:13:17,235 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 124, in get_preference_extractor + model_path=spec_dict["local_path"], + ~~~~~~~~~^^^^^^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable + +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.20s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.59s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.89s/it] +2025-12-25 09:13:47,069 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 09:13:47,070 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 124, in get_preference_extractor + model_path=spec_dict["local_path"], + ~~~~~~~~~^^^^^^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable + +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.87s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.57s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.53s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.20s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.64s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.21s/it][2025-12-25T09:14:07.340] error: *** JOB 14355901 ON gpua036 CANCELLED AT 2025-12-25T09:14:07 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355902.err b/collaborativeagents/slurm/logs/run_expts_a100_14355902.err new file mode 100644 index 0000000..c0437af --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355902.err @@ -0,0 +1,185 @@ +2025-12-25 09:14:36,278 - INFO - Loaded dataset: math-500 +2025-12-25 09:14:36,576 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:14:36,577 - INFO - Running method: vanilla +2025-12-25 09:14:36,577 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.14s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.32s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:05, 5.91s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 3.76s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.62s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:06<00:25, 6.47s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.69s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.16s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.17s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 3.31s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 4.01s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.29s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.54s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:07, 2.66s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.96s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.60s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.05s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.34s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.16s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.75s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.38s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.32s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.54s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:07, 2.66s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.99s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.20s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.47s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.65s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.64s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:14<00:04, 4.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.11s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.86s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.95s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.20s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.08s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.37s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.62s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:07, 7.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.04s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:07, 2.61s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.34s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.54s/it] +2025-12-25 09:18:54,429 - INFO - Profile 2/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:06, 3.47s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.58s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.91s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.42s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.86s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.62s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.43s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.96s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.26s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.45s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 3.00s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.78s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.12s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.80s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.19s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.50s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.98s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.80s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.87s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.46s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.97s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.80s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.73s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.51s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.01s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.73s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.83s/it] +2025-12-25 09:22:06,817 - ERROR - Error in session: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.77 GiB is allocated by PyTorch, and 210.60 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +2025-12-25 09:22:06,836 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 123, in get_preference_extractor + return QwenRuleExtractor( + ^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/preference_extractor/rule_extractor.py", line 36, in __init__ + self.model = AutoModelForCausalLM.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 604, in from_pretrained + return model_class.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 277, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5048, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5468, in _load_pretrained_model + _error_msgs, disk_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 843, in load_shard_file + disk_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 770, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + ^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.77 GiB is allocated by PyTorch, and 210.60 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) + +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.61s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.05s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.34s/it] +2025-12-25 09:22:24,976 - INFO - Based on the current allocation process, no modules could be assigned to the following devices due to insufficient memory: + - 0: 2484944896 bytes required +These minimum requirements are specific to this allocation attempt and may vary. Consider increasing the available memory for these devices to at least the specified minimum, or adjusting the model config. +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.51s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.98s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.65s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.79s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.86s/it] +2025-12-25 09:22:39,682 - INFO - Based on the current allocation process, no modules could be assigned to the following devices due to insufficient memory: + - 0: 560343040 bytes required +These minimum requirements are specific to this allocation attempt and may vary. Consider increasing the available memory for these devices to at least the specified minimum, or adjusting the model config. +2025-12-25 09:22:44,762 - ERROR - Error in session: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.76 GiB is allocated by PyTorch, and 219.26 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +2025-12-25 09:22:44,764 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 123, in get_preference_extractor + return QwenRuleExtractor( + ^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/preference_extractor/rule_extractor.py", line 36, in __init__ + self.model = AutoModelForCausalLM.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 604, in from_pretrained + return model_class.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 277, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5048, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5468, in _load_pretrained_model + _error_msgs, disk_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 843, in load_shard_file + disk_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 770, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + ^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.76 GiB is allocated by PyTorch, and 219.26 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) + +2025-12-25 09:22:44,854 - INFO - Profile 3/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.75s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.21s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.97s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.30s/it] +2025-12-25 09:23:02,714 - INFO - Based on the current allocation process, no modules could be assigned to the following devices due to insufficient memory: + - 0: 2484944896 bytes required +These minimum requirements are specific to this allocation attempt and may vary. Consider increasing the available memory for these devices to at least the specified minimum, or adjusting the model config. +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.44s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.02s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.29s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.70s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.07s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.80s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.35s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.44s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.95s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.44s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.26s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.85s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.77s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.68s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.09s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.57s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.01s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.51s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.78s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.06s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.40s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.01s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.31s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.51s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.67s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.90s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 6.00s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.69s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.17s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.15s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.32s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.67s/it] +2025-12-25 09:26:54,931 - INFO - Profile 4/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.57s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.94s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.14s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.34s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.29s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.56s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.85s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.20s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.13s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.77s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.29s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.79s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.17s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.62s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.83s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.63s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.08s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.49s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.94s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.91s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.45s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.14s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.83s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.91s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.64s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.08s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.42s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.63s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.52s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.74s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 3.00s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.15s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:15, 3.79s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.92s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:11<00:07, 3.98s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.40s/it] +2025-12-25 09:30:47,307 - INFO - Profile 5/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.89s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.52s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.59s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.84s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.73s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.24s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.34s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.53s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.72s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.97s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.26s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.86s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.33s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.84s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.56s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.88s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.23s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.01s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.46s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.98s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.41s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.82s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.32s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:16, 5.54s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:08, 4.42s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.90s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.34s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.68s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.29s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:04, 4.12s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.13s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.05s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.68s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.26s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.82s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.24s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.07s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.93s/it] +2025-12-25 09:34:45,922 - INFO - Profile 6/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.60s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 4.00s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:07, 7.07s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.34s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.77s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.66s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.12s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.72s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.90s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.50s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.56s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.52s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.88s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.59s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.39s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.19s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.89s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.60s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.01s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.64s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 3.00s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.99s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.53s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.81s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.56s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.05s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.73s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.14s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.58s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.83s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.89s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.93s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.56s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.83s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.79s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.23s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.83s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.32s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.74s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:06, 3.38s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.76s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.14s/it] +2025-12-25 09:38:42,663 - INFO - Profile 7/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.81s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.75s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.03s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.11s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.41s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.14s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.71s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.58s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.93s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.18s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.02s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.28s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.57s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.28s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.99s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.38s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.96s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.19s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.81s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.26s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.15s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.43s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.20s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.03s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.46s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.07s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.32s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.89s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.11s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.58s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.50s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.10s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.85s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.21s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.88s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.32s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.91s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.72s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.02s/it] +2025-12-25 09:42:41,214 - INFO - Profile 8/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.93s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.22s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.97s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.48s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.81s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.60s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.26s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.71s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.03s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.27s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.33s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.22s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.83s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.39s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.86s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.25s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.07s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.54s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.47s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.82s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.32s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.65s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.09s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.55s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.89s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.11s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.51s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.95s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.28s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.70s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.19s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.54s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.86s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.55s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.90s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.69s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.78s/it][2025-12-25T09:46:14.933] error: *** JOB 14355902 ON gpua065 CANCELLED AT 2025-12-25T09:46:14 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355919.err b/collaborativeagents/slurm/logs/run_expts_a100_14355919.err new file mode 100644 index 0000000..0a2aab0 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355919.err @@ -0,0 +1,97 @@ +2025-12-25 09:48:42,727 - INFO - Loaded dataset: math-500 +2025-12-25 09:48:42,889 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:48:42,890 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.41s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.19s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.44s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.43s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.40s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:06, 3.41s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.87s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.84s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.12s/it] +2025-12-25 09:49:25,411 - INFO - Profile 1/20 +2025-12-25 09:50:33,069 - INFO - Profile 2/20 +2025-12-25 09:51:32,211 - INFO - Profile 3/20 +2025-12-25 09:52:38,264 - INFO - Profile 4/20 +2025-12-25 09:53:45,537 - INFO - Profile 5/20 +2025-12-25 09:54:50,435 - INFO - Profile 6/20 +2025-12-25 09:55:54,476 - INFO - Profile 7/20 +2025-12-25 09:56:57,290 - INFO - Profile 8/20 +2025-12-25 09:57:59,489 - INFO - Profile 9/20 +2025-12-25 09:59:06,837 - INFO - Profile 10/20 +2025-12-25 10:00:16,031 - INFO - Profile 11/20 +2025-12-25 10:01:22,406 - INFO - Profile 12/20 +2025-12-25 10:02:29,213 - INFO - Profile 13/20 +2025-12-25 10:03:33,203 - INFO - Profile 14/20 +2025-12-25 10:04:37,973 - INFO - Profile 15/20 +2025-12-25 10:05:48,613 - INFO - Profile 16/20 +2025-12-25 10:06:55,828 - INFO - Profile 17/20 +2025-12-25 10:08:00,583 - INFO - Profile 18/20 +2025-12-25 10:09:08,100 - INFO - Profile 19/20 +2025-12-25 10:10:14,307 - INFO - Profile 20/20 +2025-12-25 10:11:19,207 - INFO - Running method: all_memory +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.04s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.63s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.27s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.08s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:21, 5.37s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:15, 5.16s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:16<00:10, 5.40s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:20<00:04, 4.91s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 3.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 4.45s/it] +2025-12-25 10:12:53,238 - INFO - Profile 1/20 +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-25 10:13:59,767 - INFO - Profile 2/20 +2025-12-25 10:15:08,683 - INFO - Profile 3/20 +2025-12-25 10:16:16,674 - INFO - Profile 4/20 +2025-12-25 10:17:27,036 - INFO - Profile 5/20 +2025-12-25 10:18:36,939 - INFO - Profile 6/20 +2025-12-25 10:19:50,147 - INFO - Profile 7/20 +2025-12-25 10:20:58,377 - INFO - Profile 8/20 +2025-12-25 10:22:04,174 - INFO - Profile 9/20 +2025-12-25 10:23:04,466 - INFO - Profile 10/20 +2025-12-25 10:23:59,551 - INFO - Profile 11/20 +2025-12-25 10:25:00,817 - INFO - Profile 12/20 +2025-12-25 10:26:12,175 - INFO - Profile 13/20 +2025-12-25 10:27:07,257 - INFO - Profile 14/20 +2025-12-25 10:28:21,198 - INFO - Profile 15/20 +2025-12-25 10:29:25,304 - INFO - Profile 16/20 +2025-12-25 10:30:33,063 - INFO - Profile 17/20 +2025-12-25 10:31:41,527 - INFO - Profile 18/20 +2025-12-25 10:32:46,068 - INFO - Profile 19/20 +2025-12-25 10:33:55,833 - INFO - Profile 20/20 +2025-12-25 10:34:52,399 - INFO - Running method: rag +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.92s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.25s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.47s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.99s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:07<00:31, 7.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:13<00:19, 6.53s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:19<00:12, 6.32s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:24<00:05, 5.94s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:26<00:00, 4.46s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:26<00:00, 5.35s/it] +2025-12-25 10:36:26,619 - INFO - Profile 1/20 +2025-12-25 10:37:35,693 - INFO - Profile 2/20 +2025-12-25 10:38:41,748 - INFO - Profile 3/20 +2025-12-25 10:39:47,539 - INFO - Profile 4/20 +2025-12-25 10:40:52,966 - INFO - Profile 5/20 +2025-12-25 10:41:55,106 - INFO - Profile 6/20 +2025-12-25 10:43:03,681 - INFO - Profile 7/20 +2025-12-25 10:44:11,999 - INFO - Profile 8/20 +2025-12-25 10:45:27,468 - INFO - Profile 9/20 +2025-12-25 10:46:29,111 - INFO - Profile 10/20 +2025-12-25 10:47:36,526 - INFO - Profile 11/20 +2025-12-25 10:48:40,778 - INFO - Profile 12/20 +2025-12-25 10:49:56,738 - INFO - Profile 13/20 +2025-12-25 10:51:03,796 - INFO - Profile 14/20 +2025-12-25 10:52:10,335 - INFO - Profile 15/20 +2025-12-25 10:53:16,174 - INFO - Profile 16/20 +2025-12-25 10:54:21,228 - INFO - Profile 17/20 +2025-12-25 10:55:26,718 - INFO - Profile 18/20 +2025-12-25 10:56:33,006 - INFO - Profile 19/20 +2025-12-25 10:57:36,404 - INFO - Profile 20/20 +2025-12-25 10:58:45,324 - INFO - Running method: rag_vector +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:22, 7.43s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.12s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.58s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.48s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:07<00:30, 7.59s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:13<00:20, 6.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:19<00:12, 6.48s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:25<00:06, 6.23s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 4.78s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 5.60s/it] +2025-12-25 10:59:46,496 - INFO - Profile 1/20 +2025-12-25 11:00:34,992 - INFO - Profile 2/20 +2025-12-25 11:01:23,029 - INFO - Profile 3/20 +2025-12-25 11:02:15,132 - INFO - Profile 4/20 +2025-12-25 11:02:57,063 - INFO - Profile 5/20 +2025-12-25 11:03:49,132 - INFO - Profile 6/20 +2025-12-25 11:04:41,128 - INFO - Profile 7/20 +2025-12-25 11:05:28,179 - INFO - Profile 8/20 +2025-12-25 11:06:17,236 - INFO - Profile 9/20 +2025-12-25 11:07:04,377 - INFO - Profile 10/20 +2025-12-25 11:07:55,209 - INFO - Profile 11/20 +2025-12-25 11:08:41,927 - INFO - Profile 12/20 +2025-12-25 11:09:33,573 - INFO - Profile 13/20 +2025-12-25 11:10:20,629 - INFO - Profile 14/20 +2025-12-25 11:11:05,039 - INFO - Profile 15/20 +2025-12-25 11:11:57,558 - INFO - Profile 16/20 +2025-12-25 11:12:45,379 - INFO - Profile 17/20 +2025-12-25 11:13:30,642 - INFO - Profile 18/20 +2025-12-25 11:14:22,279 - INFO - Profile 19/20 +2025-12-25 11:15:11,250 - INFO - Profile 20/20 +2025-12-25 11:16:03,557 - INFO - Report saved to ../results/full_20251225_094835/20251225_094842/report.md diff --git a/collaborativeagents/slurm/logs/run_multiturn_14357110.err b/collaborativeagents/slurm/logs/run_multiturn_14357110.err new file mode 100644 index 0000000..53553f1 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_multiturn_14357110.err @@ -0,0 +1,9 @@ +2025-12-25 21:55:51,030 - INFO - Loaded dataset: math-500 +2025-12-25 21:55:51,242 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 21:55:51,243 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.28s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.08s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.12s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:18, 4.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:13, 4.55s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.55s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.38s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.87s/it] +2025-12-25 21:56:39,760 - INFO - Profile 1/5 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.03s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.01s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 5.80s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.43s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.98s/it] +[2025-12-25T21:58:59.259] error: *** JOB 14357110 ON gpua016 CANCELLED AT 2025-12-25T21:58:59 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_multiturn_14357122.err b/collaborativeagents/slurm/logs/run_multiturn_14357122.err new file mode 100644 index 0000000..b79f0e7 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_multiturn_14357122.err @@ -0,0 +1,98 @@ +2025-12-25 22:08:21,888 - INFO - Loaded dataset: math-500 +2025-12-25 22:08:22,033 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 22:08:22,034 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.85s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.08s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.69s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.60s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:15, 3.81s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.25s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:06, 3.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.86s/it] +2025-12-25 22:08:59,678 - INFO - Profile 1/5 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:10<00:32, 10.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:15<00:14, 7.08s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:05, 5.68s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.06s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.23s/it] +2025-12-25 22:12:43,460 - WARNING - User agent failed to respond at turn 4 +2025-12-25 22:14:39,792 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:14:39,793 - INFO - Profile 2/5 +2025-12-25 22:17:30,565 - INFO - Profile 3/5 +2025-12-25 22:20:00,571 - INFO - Profile 4/5 +2025-12-25 22:23:05,146 - WARNING - User agent failed to respond at turn 4 +2025-12-25 22:23:35,365 - INFO - Profile 5/5 +2025-12-25 22:26:59,994 - INFO - Running method: all_memory +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:08<00:24, 8.06s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:10, 5.48s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:20<00:07, 7.15s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.33s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:22, 5.64s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:12, 4.14s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:03, 3.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 2.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.68s/it] +2025-12-25 22:28:23,340 - INFO - Profile 1/5 +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-25 22:29:58,580 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:30:39,359 - INFO - Profile 2/5 +2025-12-25 22:34:23,370 - INFO - Profile 3/5 +2025-12-25 22:35:04,289 - WARNING - User agent failed to respond at turn 2 +2025-12-25 22:35:30,064 - WARNING - User agent failed to respond at turn 2 +2025-12-25 22:36:33,412 - WARNING - User agent failed to respond at turn 6 +2025-12-25 22:36:33,412 - INFO - Profile 4/5 +2025-12-25 22:38:38,658 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:39:23,955 - INFO - Profile 5/5 +2025-12-25 22:42:19,402 - INFO - Running method: rag +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:20, 6.89s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.09s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.42s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.16s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.98s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:06<00:24, 6.16s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:11<00:16, 5.55s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:17<00:11, 5.90s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:22<00:05, 5.37s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:23<00:00, 4.06s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:23<00:00, 4.78s/it] +2025-12-25 22:43:31,077 - INFO - Profile 1/5 +2025-12-25 22:46:19,125 - INFO - Profile 2/5 +2025-12-25 22:49:18,368 - INFO - Profile 3/5 +2025-12-25 22:53:00,495 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:53:00,497 - INFO - Profile 4/5 +2025-12-25 22:54:01,784 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:58:07,157 - INFO - Profile 5/5 +2025-12-25 22:58:54,351 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:59:40,507 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:00:10,569 - INFO - Running method: rag_vector +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:22, 7.37s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.14s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 3.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.74s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:06<00:25, 6.35s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:12, 4.31s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.22s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.18s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.16s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.89s/it] +2025-12-25 23:01:19,774 - INFO - Profile 1/5 +2025-12-25 23:03:56,207 - INFO - Profile 2/5 +2025-12-25 23:06:30,341 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:06:30,342 - INFO - Profile 3/5 +2025-12-25 23:09:50,352 - WARNING - User agent failed to respond at turn 7 +2025-12-25 23:11:12,291 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:11:12,293 - INFO - Profile 4/5 +2025-12-25 23:14:00,507 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:15:21,185 - INFO - Profile 5/5 +2025-12-25 23:17:09,189 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:17:38,489 - INFO - Running method: contextual +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.83s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:11, 5.97s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 5.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.31s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.89s/it] +2025-12-25 23:17:58,530 - INFO - Profile 1/5 +2025-12-25 23:22:15,614 - WARNING - User agent failed to respond at turn 5 +2025-12-25 23:23:01,495 - INFO - Profile 2/5 +2025-12-25 23:26:21,325 - WARNING - User agent failed to respond at turn 12 +2025-12-25 23:26:21,326 - INFO - Profile 3/5 +2025-12-25 23:29:17,191 - WARNING - User agent failed to respond at turn 7 +2025-12-25 23:30:41,180 - INFO - Profile 4/5 +2025-12-25 23:31:24,578 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:33:26,694 - WARNING - User agent failed to respond at turn 6 +2025-12-25 23:35:25,025 - WARNING - User agent failed to respond at turn 6 +2025-12-25 23:35:25,025 - INFO - Profile 5/5 +2025-12-25 23:36:14,963 - WARNING - User agent failed to respond at turn 3 +2025-12-25 23:37:33,084 - WARNING - User agent failed to respond at turn 3 +2025-12-25 23:38:11,089 - INFO - Running method: reflection +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.99s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.61s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.87s/it] +2025-12-25 23:38:27,190 - INFO - Profile 1/5 +2025-12-25 23:41:28,520 - WARNING - User agent failed to respond at turn 3 +2025-12-25 23:42:37,103 - INFO - Profile 2/5 +2025-12-25 23:46:33,054 - WARNING - User agent failed to respond at turn 7 +2025-12-25 23:46:46,658 - INFO - Profile 3/5 +2025-12-25 23:49:40,906 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:50:58,786 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:51:12,246 - INFO - Profile 4/5 +2025-12-25 23:52:14,159 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:55:01,535 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:56:57,317 - INFO - Profile 5/5 +2025-12-25 23:58:27,891 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:59:29,746 - INFO - Running method: reflection_grpo +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:21, 7.21s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.18s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:05, 5.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.99s/it] +2025-12-25 23:59:50,260 - INFO - Profile 1/5 +2025-12-26 00:00:31,696 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:03:33,202 - INFO - Profile 2/5 +2025-12-26 00:06:53,817 - INFO - Profile 3/5 +2025-12-26 00:10:53,169 - WARNING - User agent failed to respond at turn 4 +2025-12-26 00:12:53,034 - WARNING - User agent failed to respond at turn 4 +2025-12-26 00:13:06,491 - INFO - Profile 4/5 +2025-12-26 00:13:59,355 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:18:16,345 - INFO - Profile 5/5 +2025-12-26 00:18:53,569 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:19:48,324 - WARNING - User agent failed to respond at turn 2 +2025-12-26 00:20:53,392 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:21:06,861 - INFO - Report saved to ../results/multiturn_test_20251225_220813/20251225_220821/report.md diff --git a/collaborativeagents/slurm/logs/test_70b_14357753.err b/collaborativeagents/slurm/logs/test_70b_14357753.err new file mode 100644 index 0000000..994c585 --- /dev/null +++ b/collaborativeagents/slurm/logs/test_70b_14357753.err @@ -0,0 +1 @@ +`torch_dtype` is deprecated! Use `dtype` instead! diff --git a/collaborativeagents/slurm/logs/test_70b_14357762.err b/collaborativeagents/slurm/logs/test_70b_14357762.err new file mode 100644 index 0000000..db6b275 --- /dev/null +++ b/collaborativeagents/slurm/logs/test_70b_14357762.err @@ -0,0 +1,24 @@ +`torch_dtype` is deprecated! Use `dtype` instead! +
Fetching 9 files: 0%| | 0/9 [00:00<?, ?it/s]
Fetching 9 files: 11%|█ | 1/9 [01:04<08:38, 64.82s/it]
Fetching 9 files: 22%|██▏ | 2/9 [01:05<03:09, 27.05s/it]
Fetching 9 files: 44%|████▍ | 4/9 [01:06<00:52, 10.45s/it]
Fetching 9 files: 78%|███████▊ | 7/9 [01:33<00:19, 9.67s/it]
Fetching 9 files: 100%|██████████| 9/9 [01:33<00:00, 10.43s/it] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:03<00:26, 3.34s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:09<00:33, 4.81s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:11<00:22, 3.73s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:17<00:22, 4.51s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:23<00:20, 5.10s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:28<00:15, 5.08s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:23<00:43, 21.52s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:26<00:15, 15.40s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:27<00:00, 10.90s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:27<00:00, 9.69s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.90s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:11, 5.91s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:15<01:02, 15.59s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:21<00:30, 10.08s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:27<00:16, 8.23s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:32<00:06, 6.79s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:34<00:00, 5.01s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:34<00:00, 6.85s/it] +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:02<00:19, 2.47s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [01:00<04:05, 35.01s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [01:05<02:07, 21.33s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [01:10<01:14, 14.81s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [01:15<00:45, 11.36s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:20<00:27, 9.25s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:30<00:19, 9.50s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:32<00:07, 7.02s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:33<00:00, 5.34s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:33<00:00, 10.43s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.44s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.72s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.15s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.87s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:21, 5.25s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:13, 4.63s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.53s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.87s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.05s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.66s/it] diff --git a/collaborativeagents/slurm/logs/test_extractor_14363568.err b/collaborativeagents/slurm/logs/test_extractor_14363568.err new file mode 100644 index 0000000..9b05d44 --- /dev/null +++ b/collaborativeagents/slurm/logs/test_extractor_14363568.err @@ -0,0 +1,3 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. diff --git a/collaborativeagents/slurm/logs/test_multiturn_14357116.err b/collaborativeagents/slurm/logs/test_multiturn_14357116.err new file mode 100644 index 0000000..91eba3b --- /dev/null +++ b/collaborativeagents/slurm/logs/test_multiturn_14357116.err @@ -0,0 +1,13 @@ +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.54s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.54s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.03s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.95s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.74s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.43s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.92s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:16, 4.24s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.81s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:11<00:07, 3.68s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.22s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.46s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.49s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.42s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.17s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.17s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.59s/it] +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 226, in <module> + results["full_session"] = test_full_session() + ^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 171, in test_full_session + adapter = create_baseline_adapter("vanilla") + ^^^^^^^^^^^^^^^^^^^^^^^ +NameError: name 'create_baseline_adapter' is not defined diff --git a/collaborativeagents/slurm/logs/test_multiturn_14357119.err b/collaborativeagents/slurm/logs/test_multiturn_14357119.err new file mode 100644 index 0000000..1a66fce --- /dev/null +++ b/collaborativeagents/slurm/logs/test_multiturn_14357119.err @@ -0,0 +1,46 @@ +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.50s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.48s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.29s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.66s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.13s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.16s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.51s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:17, 4.31s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:12, 4.11s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.30s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.30s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.31s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:10, 5.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:14<00:04, 4.67s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.74s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.10s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.62s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.74s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:18, 4.50s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.86s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.03s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.82s/it] +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 227, in <module> + results["full_session"] = test_full_session() + ^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 173, in test_full_session + adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 295, in _load_models + self._reranker = Qwen3Reranker( + ^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/reranker/qwen3_reranker.py", line 26, in __init__ + self.model = AutoModelForCausalLM.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 604, in from_pretrained + return model_class.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 277, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5048, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5468, in _load_pretrained_model + _error_msgs, disk_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 843, in load_shard_file + disk_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 770, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + ^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 32.00 MiB. GPU 1 has a total capacity of 39.49 GiB of which 30.31 MiB is free. Including non-PyTorch memory, this process has 39.46 GiB memory in use. Of the allocated memory 38.87 GiB is allocated by PyTorch, and 87.01 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) diff --git a/collaborativeagents/slurm/logs/vanilla_14360419.err b/collaborativeagents/slurm/logs/vanilla_14360419.err new file mode 100644 index 0000000..467f4f7 --- /dev/null +++ b/collaborativeagents/slurm/logs/vanilla_14360419.err @@ -0,0 +1,138 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:32:07,469 - INFO - Loaded dataset: mmlu +2025-12-27 01:32:07,469 - INFO - Loaded dataset: aime +2025-12-27 01:32:07,469 - INFO - Loaded dataset: math-hard +2025-12-27 01:32:07,469 - INFO - Loaded dataset: humaneval +2025-12-27 01:32:07,504 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:32:07,505 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.81s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:11, 5.63s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.23s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.05s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.74s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:19, 5.00s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.76s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.44s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 4.00s/it] +2025-12-27 01:32:57,620 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:11<01:28, 11.01s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:21<01:15, 10.75s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:33<01:07, 11.20s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:43<00:54, 10.89s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [01:02<00:54, 13.70s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:12<00:37, 12.53s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:21<00:22, 11.22s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:28<00:09, 9.92s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:32<00:00, 8.13s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:32<00:00, 10.28s/it] +2025-12-27 01:39:14,965 - WARNING - User agent failed to respond at turn 3 +2025-12-27 01:45:16,597 - WARNING - User agent failed to respond at turn 2 +2025-12-27 02:07:27,447 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:10:51,780 - WARNING - User agent failed to respond at turn 4 +2025-12-27 02:21:16,753 - INFO - Profile 2/30 +2025-12-27 02:56:57,401 - WARNING - User agent failed to respond at turn 7 +2025-12-27 03:08:20,574 - WARNING - User agent failed to respond at turn 5 +2025-12-27 03:13:14,352 - INFO - Profile 3/30 +2025-12-27 03:25:06,846 - WARNING - User agent failed to respond at turn 4 +2025-12-27 03:31:15,270 - WARNING - User agent failed to respond at turn 3 +2025-12-27 03:51:13,452 - WARNING - User agent failed to respond at turn 5 +2025-12-27 03:53:58,854 - WARNING - User agent failed to respond at turn 3 +2025-12-27 03:56:56,391 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:04:16,140 - INFO - Profile 4/30 +2025-12-27 04:11:00,592 - WARNING - User agent failed to respond at turn 5 +2025-12-27 04:25:11,672 - WARNING - User agent failed to respond at turn 6 +2025-12-27 04:28:01,010 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:35:32,474 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:58:18,335 - WARNING - User agent failed to respond at turn 4 +2025-12-27 05:01:09,794 - WARNING - User agent failed to respond at turn 3 +2025-12-27 05:10:18,666 - INFO - Profile 5/30 +2025-12-27 05:49:33,423 - INFO - Profile 6/30 +2025-12-27 06:18:07,136 - WARNING - User agent failed to respond at turn 3 +2025-12-27 06:40:37,716 - INFO - Profile 7/30 +2025-12-27 06:52:19,840 - WARNING - User agent failed to respond at turn 4 +2025-12-27 07:37:53,426 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:40:19,372 - INFO - Profile 8/30 +2025-12-27 07:51:25,858 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:07:46,046 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:13:28,244 - WARNING - User agent failed to respond at turn 4 +2025-12-27 08:16:50,128 - WARNING - User agent failed to respond at turn 5 +2025-12-27 08:43:58,910 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:43:58,912 - INFO - Profile 9/30 +2025-12-27 08:52:45,466 - WARNING - User agent failed to respond at turn 2 +2025-12-27 09:28:02,037 - INFO - Profile 10/30 +2025-12-27 09:43:20,171 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:15:12,819 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:29:30,033 - INFO - Profile 11/30 +2025-12-27 11:22:03,526 - INFO - Profile 12/30 +2025-12-27 12:01:00,590 - WARNING - User agent failed to respond at turn 3 +2025-12-27 12:13:47,494 - INFO - Profile 13/30 +2025-12-27 12:39:42,451 - WARNING - User agent failed to respond at turn 5 +2025-12-27 12:44:54,076 - WARNING - User agent failed to respond at turn 2 +2025-12-27 12:56:38,326 - INFO - Profile 14/30 +2025-12-27 13:04:45,544 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:23:13,051 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:36:48,889 - INFO - Profile 15/30 +2025-12-27 14:18:21,034 - INFO - Profile 16/30 +2025-12-27 15:03:24,448 - INFO - Profile 17/30 +2025-12-27 15:05:16,405 - WARNING - User agent failed to respond at turn 2 +2025-12-27 15:09:02,780 - WARNING - User agent failed to respond at turn 4 +2025-12-27 15:22:15,600 - WARNING - User agent failed to respond at turn 5 +2025-12-27 15:32:10,486 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:40:26,743 - WARNING - User agent failed to respond at turn 2 +2025-12-27 15:43:30,526 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:57:53,732 - INFO - Profile 18/30 +2025-12-27 16:21:37,879 - WARNING - User agent failed to respond at turn 0 +2025-12-27 16:32:11,066 - WARNING - User agent failed to respond at turn 5 +2025-12-27 16:43:57,955 - WARNING - User agent failed to respond at turn 0 +2025-12-27 16:43:57,957 - INFO - Profile 19/30 +2025-12-27 17:09:46,077 - WARNING - User agent failed to respond at turn 3 +2025-12-27 17:20:38,532 - WARNING - User agent failed to respond at turn 4 +2025-12-27 17:36:07,824 - WARNING - User agent failed to respond at turn 4 +2025-12-27 17:36:07,826 - INFO - Profile 20/30 +2025-12-27 17:56:48,744 - WARNING - User agent failed to respond at turn 3 +2025-12-27 18:16:09,433 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:27:50,852 - WARNING - User agent failed to respond at turn 5 +2025-12-27 18:33:04,262 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:48:42,005 - INFO - Profile 21/30 +2025-12-27 19:01:49,962 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:04:32,307 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:11:14,252 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:49:58,335 - WARNING - User agent failed to respond at turn 5 +2025-12-27 19:57:43,221 - WARNING - User agent failed to respond at turn 6 +2025-12-27 20:00:05,573 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:00:05,574 - INFO - Profile 22/30 +2025-12-27 20:39:06,239 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:47:11,767 - WARNING - User agent failed to respond at turn 9 +2025-12-27 21:00:45,614 - INFO - Profile 23/30 +2025-12-27 21:03:59,797 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:02:44,465 - WARNING - User agent failed to respond at turn 4 +2025-12-27 22:02:44,467 - INFO - Profile 24/30 +2025-12-27 22:14:01,188 - WARNING - User agent failed to respond at turn 4 +2025-12-27 22:44:39,432 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:57:20,908 - WARNING - User agent failed to respond at turn 2 +2025-12-27 22:58:16,593 - INFO - Profile 25/30 +2025-12-27 23:18:08,405 - WARNING - User agent failed to respond at turn 13 +2025-12-27 23:29:17,609 - WARNING - User agent failed to respond at turn 2 +2025-12-27 23:50:46,712 - INFO - Profile 26/30 +2025-12-28 00:02:31,997 - WARNING - User agent failed to respond at turn 4 +2025-12-28 00:10:38,451 - WARNING - User agent failed to respond at turn 2 +2025-12-28 00:37:35,648 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:53:32,683 - INFO - Profile 27/30 +2025-12-28 01:00:46,162 - WARNING - User agent failed to respond at turn 6 +2025-12-28 01:27:50,250 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:50:48,239 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:58:29,456 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:08:15,289 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:08:15,290 - INFO - Profile 28/30 +2025-12-28 02:26:56,690 - WARNING - User agent failed to respond at turn 2 +2025-12-28 02:32:37,000 - WARNING - User agent failed to respond at turn 4 +2025-12-28 03:11:21,975 - WARNING - User agent failed to respond at turn 4 +2025-12-28 03:11:21,977 - INFO - Profile 29/30 +2025-12-28 03:19:27,078 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:05:06,853 - WARNING - User agent failed to respond at turn 5 +2025-12-28 04:22:54,056 - INFO - Profile 30/30 +2025-12-28 04:26:11,853 - WARNING - User agent failed to respond at turn 4 +2025-12-28 04:59:46,155 - WARNING - User agent failed to respond at turn 5 +2025-12-28 05:09:22,444 - INFO - Report saved to ../results/vanilla_20251227_013200/20251227_013207/report.md diff --git a/collaborativeagents/slurm/logs/vllm_bench_14367333.err b/collaborativeagents/slurm/logs/vllm_bench_14367333.err new file mode 100644 index 0000000..ced0fa9 --- /dev/null +++ b/collaborativeagents/slurm/logs/vllm_bench_14367333.err @@ -0,0 +1,6 @@ +/u/yurenh2/.local/lib/python3.9/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [02:34<07:43, 154.66s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [03:10<02:49, 84.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [03:41<01:00, 60.27s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [04:00<00:00, 43.81s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [04:00<00:00, 60.08s/it] +/var/spool/slurmd/job14367333/slurm_script: line 39: 3292815 Killed python scripts/benchmark_inference.py --mode transformers --model $MODEL_8B -n 10 +[2025-12-29T04:01:31.106] error: Detected 1 oom_kill event in StepId=14367333.batch. Some of the step tasks have been OOM Killed. diff --git a/collaborativeagents/slurm/logs/vllm_bench_70b_8b_14367370.err b/collaborativeagents/slurm/logs/vllm_bench_70b_8b_14367370.err new file mode 100644 index 0000000..91e66ef --- /dev/null +++ b/collaborativeagents/slurm/logs/vllm_bench_70b_8b_14367370.err @@ -0,0 +1,180 @@ +
Fetching 19 files: 0%| | 0/19 [00:00<?, ?it/s]
Fetching 19 files: 5%|▌ | 1/19 [00:00<00:08, 2.19it/s]
Fetching 19 files: 37%|███▋ | 7/19 [01:23<02:29, 12.48s/it]
Fetching 19 files: 47%|████▋ | 9/19 [01:35<01:44, 10.44s/it]
Fetching 19 files: 53%|█████▎ | 10/19 [01:36<01:19, 8.86s/it]
Fetching 19 files: 100%|██████████| 19/19 [01:36<00:00, 5.08s/it] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(APIServer pid=3643829)[0;0m
Parse safetensors files: 0%| | 0/9 [00:00<?, ?it/s]
Parse safetensors files: 11%|█ | 1/9 [00:00<00:01, 5.28it/s]
Parse safetensors files: 100%|██████████| 9/9 [00:00<00:00, 46.86it/s] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:07<00:22, 7.57s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/9 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:10<00:09, 4.64s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 11% Completed | 1/9 [00:02<00:21, 2.72s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 22% Completed | 2/9 [00:07<00:26, 3.72s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:18<00:06, 6.17s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 33% Completed | 3/9 [00:11<00:23, 3.91s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 44% Completed | 4/9 [00:15<00:21, 4.21s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:26<00:00, 6.99s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:26<00:00, 6.60s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 56% Completed | 5/9 [00:21<00:18, 4.63s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 67% Completed | 6/9 [00:26<00:14, 4.74s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 78% Completed | 7/9 [00:30<00:09, 4.56s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 89% Completed | 8/9 [00:34<00:04, 4.34s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 9/9 [00:36<00:00, 3.69s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 9/9 [00:36<00:00, 4.06s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:04, 10.42it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:04, 10.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:04, 10.82it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:03, 11.01it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 20%|█▉ | 10/51 [00:00<00:03, 11.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 24%|██▎ | 12/51 [00:01<00:03, 11.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:01<00:03, 11.72it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 31%|███▏ | 16/51 [00:01<00:02, 11.71it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 35%|███▌ | 18/51 [00:01<00:02, 11.92it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:02, 12.06it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 43%|████▎ | 22/51 [00:01<00:02, 11.85it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 24/51 [00:02<00:02, 12.01it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:02<00:02, 11.88it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 28/51 [00:02<00:01, 12.10it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 59%|█████▉ | 30/51 [00:02<00:01, 11.24it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:02<00:01, 11.74it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 67%|██████▋ | 34/51 [00:02<00:01, 11.56it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████ | 36/51 [00:03<00:01, 11.78it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:03<00:01, 11.58it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 78%|███████▊ | 40/51 [00:03<00:00, 11.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 42/51 [00:03<00:00, 11.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:03<00:00, 11.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 90%|█████████ | 46/51 [00:03<00:00, 11.82it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 94%|█████████▍| 48/51 [00:04<00:00, 12.05it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:04<00:00, 12.14it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:04<00:00, 11.63it/s] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:05, 6.07it/s]
Capturing CUDA graphs (decode, FULL): 9%|▊ | 3/35 [00:00<00:03, 9.53it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:03, 9.54it/s]
Capturing CUDA graphs (decode, FULL): 17%|█▋ | 6/35 [00:00<00:02, 10.63it/s]
Capturing CUDA graphs (decode, FULL): 23%|██▎ | 8/35 [00:00<00:02, 10.81it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:02, 11.04it/s]
Capturing CUDA graphs (decode, FULL): 34%|███▍ | 12/35 [00:01<00:02, 11.24it/s]
Capturing CUDA graphs (decode, FULL): 40%|████ | 14/35 [00:01<00:01, 11.39it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:01<00:01, 11.39it/s]
Capturing CUDA graphs (decode, FULL): 51%|█████▏ | 18/35 [00:01<00:01, 11.46it/s]
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 20/35 [00:01<00:01, 11.36it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:02<00:01, 11.34it/s]
Capturing CUDA graphs (decode, FULL): 69%|██████▊ | 24/35 [00:02<00:00, 11.51it/s]
Capturing CUDA graphs (decode, FULL): 74%|███████▍ | 26/35 [00:02<00:00, 11.47it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:02<00:00, 11.52it/s]
Capturing CUDA graphs (decode, FULL): 86%|████████▌ | 30/35 [00:02<00:00, 11.58it/s]
Capturing CUDA graphs (decode, FULL): 91%|█████████▏| 32/35 [00:02<00:00, 11.47it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:03<00:00, 11.55it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:03<00:00, 11.22it/s] +[0;36m(APIServer pid=3643830)[0;0m INFO: Started server process [3643830] +[0;36m(APIServer pid=3643830)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=3643830)[0;0m INFO: Application startup complete. +[0;36m(EngineCore_DP0 pid=3644257)[0;0m Process EngineCore_DP0: +[0;36m(EngineCore_DP0 pid=3644257)[0;0m Traceback (most recent call last): +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.run() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/multiprocessing/process.py", line 108, in run +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self._target(*self._args, **self._kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise e +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +[0;36m(EngineCore_DP0 pid=3644257)[0;0m engine_core = EngineCoreProc(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m super().__init__( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 240, in _initialize_kv_caches +[0;36m(EngineCore_DP0 pid=3644257)[0;0m available_gpu_memory = self.model_executor.determine_available_memory() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/executor/abstract.py", line 126, in determine_available_memory +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self.collective_rpc("determine_available_memory") +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/executor/uniproc_executor.py", line 75, in collective_rpc +[0;36m(EngineCore_DP0 pid=3644257)[0;0m result = run_method(self.driver_worker, method, args, kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/serial_utils.py", line 461, in run_method +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return func(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return func(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/worker/gpu_worker.py", line 340, in determine_available_memory +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.model_runner.profile_run() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4474, in profile_run +[0;36m(EngineCore_DP0 pid=3644257)[0;0m hidden_states, last_hidden_states = self._dummy_run( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return func(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run +[0;36m(EngineCore_DP0 pid=3644257)[0;0m outputs = self.model( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self.runnable(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._call_impl(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return forward_call(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/model_executor/models/llama.py", line 623, in forward +[0;36m(EngineCore_DP0 pid=3644257)[0;0m model_output = self.model( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/decorators.py", line 526, in __call__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m output = TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/wrapper.py", line 218, in __call__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._call_with_optional_nvtx_range( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/wrapper.py", line 109, in _call_with_optional_nvtx_range +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return callable_fn(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 845, in compile_wrapper +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1 +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 990, in _compile_fx_inner +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise InductorError(e, currentframe()).with_traceback( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 974, in _compile_fx_inner +[0;36m(EngineCore_DP0 pid=3644257)[0;0m mb_compiled_graph = fx_codegen_and_compile( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 1695, in fx_codegen_and_compile +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 1505, in codegen_and_compile +[0;36m(EngineCore_DP0 pid=3644257)[0;0m compiled_module = graph.compile_to_module() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/graph.py", line 2319, in compile_to_module +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._compile_to_module() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/graph.py", line 2325, in _compile_to_module +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/graph.py", line 2271, in codegen +[0;36m(EngineCore_DP0 pid=3644257)[0;0m result = self.wrapper_code.generate(self.is_inference) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/codegen/wrapper.py", line 1552, in generate +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._generate(is_inference) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/codegen/wrapper.py", line 1615, in _generate +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.generate_and_run_autotune_block() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/codegen/wrapper.py", line 1695, in generate_and_run_autotune_block +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise RuntimeError(f"Failed to run autotuning code block: {e}") from e +[0;36m(EngineCore_DP0 pid=3644257)[0;0m torch._inductor.exc.InductorError: RuntimeError: Failed to run autotuning code block: CUDA out of memory. Tried to allocate 1.96 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.86 GiB is free. Including non-PyTorch memory, this process has 37.63 GiB memory in use. Of the allocated memory 37.11 GiB is allocated by PyTorch, and 20.90 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank0]:[W1229 07:04:13.476894153 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[0;36m(APIServer pid=3643829)[0;0m Traceback (most recent call last): +[0;36m(APIServer pid=3643829)[0;0m File "<frozen runpy>", line 198, in _run_module_as_main +[0;36m(APIServer pid=3643829)[0;0m File "<frozen runpy>", line 88, in _run_code +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 1469, in <module> +[0;36m(APIServer pid=3643829)[0;0m uvloop.run(run_server(args)) +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/uvloop/__init__.py", line 92, in run +[0;36m(APIServer pid=3643829)[0;0m return runner.run(wrapper()) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/asyncio/runners.py", line 118, in run +[0;36m(APIServer pid=3643829)[0;0m return self._loop.run_until_complete(task) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/uvloop/__init__.py", line 48, in wrapper +[0;36m(APIServer pid=3643829)[0;0m return await main +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server +[0;36m(APIServer pid=3643829)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker +[0;36m(APIServer pid=3643829)[0;0m async with build_async_engine_client( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/contextlib.py", line 210, in __aenter__ +[0;36m(APIServer pid=3643829)[0;0m return await anext(self.gen) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client +[0;36m(APIServer pid=3643829)[0;0m async with build_async_engine_client_from_engine_args( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/contextlib.py", line 210, in __aenter__ +[0;36m(APIServer pid=3643829)[0;0m return await anext(self.gen) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args +[0;36m(APIServer pid=3643829)[0;0m async_llm = AsyncLLM.from_vllm_config( +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config +[0;36m(APIServer pid=3643829)[0;0m return cls( +[0;36m(APIServer pid=3643829)[0;0m ^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__ +[0;36m(APIServer pid=3643829)[0;0m self.engine_core = EngineCoreClient.make_async_mp_client( +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client +[0;36m(APIServer pid=3643829)[0;0m return AsyncMPClient(*client_args) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__ +[0;36m(APIServer pid=3643829)[0;0m super().__init__( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 477, in __init__ +[0;36m(APIServer pid=3643829)[0;0m with launch_core_engines(vllm_config, executor_class, log_stats) as ( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/contextlib.py", line 144, in __exit__ +[0;36m(APIServer pid=3643829)[0;0m next(self.gen) +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/utils.py", line 903, in launch_core_engines +[0;36m(APIServer pid=3643829)[0;0m wait_for_engine_startup( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/utils.py", line 960, in wait_for_engine_startup +[0;36m(APIServer pid=3643829)[0;0m raise RuntimeError( +[0;36m(APIServer pid=3643829)[0;0m RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +[2025-12-29T07:04:21.056] error: *** JOB 14367370 ON gpua051 CANCELLED AT 2025-12-29T07:04:21 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/vllm_only_14367345.err b/collaborativeagents/slurm/logs/vllm_only_14367345.err new file mode 100644 index 0000000..8713769 --- /dev/null +++ b/collaborativeagents/slurm/logs/vllm_only_14367345.err @@ -0,0 +1 @@ +/usr/bin/python: Error while finding module specification for 'vllm.entrypoints.openai.api_server' (ModuleNotFoundError: No module named 'vllm') diff --git a/collaborativeagents/slurm/rag_15640364.err b/collaborativeagents/slurm/rag_15640364.err new file mode 100644 index 0000000..9bd4347 --- /dev/null +++ b/collaborativeagents/slurm/rag_15640364.err @@ -0,0 +1,498 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:03<00:10, 3.42s/it] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:04<00:04, 2.31s/it] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:08<00:02, 2.81s/it] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:11<00:00, 3.11s/it] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:11<00:00, 2.98s/it] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 14.80it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:02, 15.78it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 16.58it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 17.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 18.35it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 19.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.82it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.41it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.93it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:01, 21.88it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 22.34it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 22.83it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 23.24it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:01<00:00, 23.67it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 24.15it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 24.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 24.87it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 21.71it/s] +[0;36m(EngineCore_DP0 pid=2060309)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 6%|▌ | 2/35 [00:00<00:01, 19.81it/s]
Capturing CUDA graphs (decode, FULL): 14%|█▍ | 5/35 [00:00<00:01, 21.73it/s]
Capturing CUDA graphs (decode, FULL): 23%|██▎ | 8/35 [00:00<00:01, 22.35it/s]
Capturing CUDA graphs (decode, FULL): 31%|███▏ | 11/35 [00:00<00:01, 22.75it/s]
Capturing CUDA graphs (decode, FULL): 40%|████ | 14/35 [00:00<00:00, 23.15it/s]
Capturing CUDA graphs (decode, FULL): 49%|████▊ | 17/35 [00:00<00:00, 23.25it/s]
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 20/35 [00:00<00:00, 23.25it/s]
Capturing CUDA graphs (decode, FULL): 66%|██████▌ | 23/35 [00:01<00:00, 23.43it/s]
Capturing CUDA graphs (decode, FULL): 74%|███████▍ | 26/35 [00:01<00:00, 23.69it/s]
Capturing CUDA graphs (decode, FULL): 83%|████████▎ | 29/35 [00:01<00:00, 23.94it/s]
Capturing CUDA graphs (decode, FULL): 91%|█████████▏| 32/35 [00:01<00:00, 24.15it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 24.35it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 23.51it/s] +[0;36m(APIServer pid=2060226)[0;0m INFO: Started server process [2060226] +[0;36m(APIServer pid=2060226)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=2060226)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-25 16:18:22,044 - INFO - Loaded dataset: math-hard +2026-01-25 16:18:22,044 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-25 16:18:22,065 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-25 16:18:22,066 - INFO - Running method: rag +2026-01-25 16:18:22,068 - INFO - Running profiles 0 to 19 (20 remaining) +2026-01-25 16:18:22,068 - INFO - Using BATCH processing (async OpenAI user) for rag +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 50%|█████ | 2/4 [00:00<00:00, 17.46it/s]
Loading checkpoint shards: 100%|██████████| 4/4 [00:00<00:00, 21.44it/s] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 60%|██████ | 3/5 [00:00<00:00, 23.39it/s]
Loading checkpoint shards: 100%|██████████| 5/5 [00:00<00:00, 30.44it/s] +2026-01-25 16:19:03,297 - INFO - Batch: 20 profiles × 5 sessions, max 8 turns/session +2026-01-25 16:19:18,037 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:18,590 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:21,590 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:21,656 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:22,080 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:23,006 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:23,637 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:24,272 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:24,283 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:26,855 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:28,332 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:28,377 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:29,027 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:29,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:31,287 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:31,769 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:32,534 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:34,567 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:45,373 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:19:46,056 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2026-01-25 16:22:11,168 - INFO - Retrying request to /chat/completions in 0.468833 seconds +2026-01-25 16:22:36,706 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:39,601 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:43,001 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:45,966 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:22:55,374 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:04,437 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:04,520 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:04,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:06,242 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:06,819 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:07,625 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:14,649 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:16,444 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:17,909 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:19,688 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:21,192 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:22,274 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:26,328 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:30,126 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:23:33,235 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:25:04,150 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:13,351 - INFO - Retrying request to /chat/completions in 0.400943 seconds +2026-01-25 16:27:34,356 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:41,190 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:45,096 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:50,714 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:57,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:27:59,320 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:04,115 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:06,508 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:15,262 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:15,901 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:16,687 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:17,340 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:17,606 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:18,310 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:18,682 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:28:27,320 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:29:10,545 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:29:18,893 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:30:29,433 - INFO - Retrying request to /chat/completions in 0.470981 seconds +2026-01-25 16:31:01,402 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:02,124 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:05,380 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:12,272 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:16,649 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:23,308 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:27,792 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:36,651 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:31:37,556 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:32:14,403 - INFO - Retrying request to /chat/completions in 0.384142 seconds +2026-01-25 16:32:54,297 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:32:56,066 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:07,881 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:11,672 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:33:22,599 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:34:05,235 - INFO - Retrying request to /chat/completions in 0.388680 seconds +2026-01-25 16:34:35,159 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:34:40,256 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:34:52,664 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:03,248 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:35:23,403 - INFO - Retrying request to /chat/completions in 0.396716 seconds +2026-01-25 16:36:17,635 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:36:24,022 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:36:51,432 - INFO - Retrying request to /chat/completions in 0.410028 seconds +2026-01-25 16:37:42,269 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:38:11,053 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:04,039 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:15,996 - INFO - Session round 1/5: 20 total, 57 sessions/hr +2026-01-25 16:39:16,000 - INFO - Retrying request to /chat/completions in 0.493739 seconds +2026-01-25 16:39:37,536 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:39,517 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:39,964 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:44,159 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:44,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:45,591 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:47,682 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:47,924 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:49,832 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:52,003 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:53,666 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:54,334 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:39:54,774 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:01,395 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:01,503 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:08,383 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:09,086 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:10,326 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:11,221 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:40:32,901 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:41:06,071 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:43:27,462 - INFO - Retrying request to /chat/completions in 0.439654 seconds +2026-01-25 16:43:40,196 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:43:42,942 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:43:43,892 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:43:59,528 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:01,642 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:06,002 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:16,757 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:18,576 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:19,017 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:21,709 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:21,795 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:23,711 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:23,744 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:25,549 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:30,187 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:30,419 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:31,975 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:32,119 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:36,759 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:44:48,951 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:45:00,847 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:45:25,455 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:45:35,194 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:45:38,548 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:45:48,820 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:45:52,875 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:08,815 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:15,303 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:18,332 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:49,801 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:49:15,808 - INFO - Retrying request to /chat/completions in 0.463476 seconds +2026-01-25 16:49:52,725 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:01,387 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:04,031 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:04,901 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:06,037 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:09,796 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:10,369 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:11,574 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:13,153 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:16,368 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:17,152 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:18,030 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:19,108 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:45,926 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:17,095 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:20,969 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:21,004 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:31,556 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:58,887 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:19,132 - INFO - Retrying request to /chat/completions in 0.389761 seconds +2026-01-25 16:54:09,464 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:54,622 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:45,535 - INFO - Retrying request to /chat/completions in 0.384185 seconds +2026-01-25 16:57:12,928 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:13,581 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:21,213 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:29,240 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:32,523 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:32,641 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:42,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:11,722 - INFO - Retrying request to /chat/completions in 0.448228 seconds +2026-01-25 16:58:35,126 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:39,095 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:41,257 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:58,827 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:03,247 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:33,195 - INFO - Retrying request to /chat/completions in 0.421950 seconds +2026-01-25 16:59:55,823 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:00,369 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:10,053 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:15,737 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:36,234 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:04,514 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:37,942 - INFO - Retrying request to /chat/completions in 0.446907 seconds +2026-01-25 17:03:08,798 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:18,098 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:37,117 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:49,812 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:54,367 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:09,491 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:15,078 - INFO - Retrying request to /chat/completions in 0.492114 seconds +2026-01-25 17:05:41,541 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:45,169 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:53,143 - INFO - Session round 2/5: 40 total, 51 sessions/hr +2026-01-25 17:05:53,147 - INFO - Retrying request to /chat/completions in 0.404344 seconds +2026-01-25 17:06:08,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:10,966 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:16,012 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:17,326 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:17,848 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:18,501 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:19,204 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:19,936 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:19,990 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:20,929 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:21,467 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:21,534 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:21,990 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:22,589 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:22,767 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:22,864 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:23,063 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:27,716 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:28,989 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:30,616 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:12,288 - INFO - Retrying request to /chat/completions in 0.449359 seconds +2026-01-25 17:09:37,149 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:43,836 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:44,645 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:46,532 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:49,653 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:49,663 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:50,792 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:50,942 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:52,545 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:53,108 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:53,585 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:55,633 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:56,534 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:00,509 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:00,683 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:01,591 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:01,819 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:10,442 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:11,484 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:21,914 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:10,766 - INFO - Retrying request to /chat/completions in 0.419919 seconds +2026-01-25 17:12:29,855 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:32,252 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:37,771 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:40,628 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:40,654 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:44,724 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:46,747 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:47,891 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:48,296 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:48,906 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:50,387 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:51,615 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:56,899 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:04,645 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:09,200 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:18,493 - INFO - Retrying request to /chat/completions in 0.459512 seconds +2026-01-25 17:14:51,326 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:52,788 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:54,049 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:57,356 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:58,023 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:01,314 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:26,429 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:56,548 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:39,025 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:35,137 - INFO - Retrying request to /chat/completions in 0.493675 seconds +2026-01-25 17:18:02,554 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:03,798 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:12,705 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:16,827 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:25,639 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:03,098 - INFO - Retrying request to /chat/completions in 0.435001 seconds +2026-01-25 17:19:35,453 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:43,815 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:47,692 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:19:57,821 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:20:22,285 - INFO - Retrying request to /chat/completions in 0.499961 seconds +2026-01-25 17:20:47,575 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:20:49,760 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:13,946 - INFO - Retrying request to /chat/completions in 0.474088 seconds +2026-01-25 17:21:45,299 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:45,423 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:08,570 - INFO - Session round 3/5: 60 total, 56 sessions/hr +2026-01-25 17:22:08,574 - INFO - Retrying request to /chat/completions in 0.487848 seconds +2026-01-25 17:22:26,921 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:29,361 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:31,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:31,598 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:31,676 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:31,707 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:33,116 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:33,739 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:35,275 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:35,542 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:35,556 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:36,745 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:37,036 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:37,609 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:37,826 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:44,531 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:45,492 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:45,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:49,264 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:54,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:25:49,005 - INFO - Retrying request to /chat/completions in 0.405132 seconds +2026-01-25 17:26:23,204 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:23,206 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:24,003 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:24,502 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:26,645 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:28,277 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:29,948 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:30,157 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:32,480 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:35,417 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:35,463 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:35,831 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:36,831 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:41,071 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:41,949 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:46,191 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:46,529 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:48,832 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:59,039 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:59,438 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:27:58,695 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:18,281 - INFO - Retrying request to /chat/completions in 0.411077 seconds +2026-01-25 17:30:41,971 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:43,771 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:46,851 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:47,178 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:50,863 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:51,165 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:56,680 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:56,825 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:57,126 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:58,097 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:01,404 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:01,608 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:09,612 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:10,218 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:13,905 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:22,803 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:24,431 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:30,341 - INFO - Retrying request to /chat/completions in 0.375630 seconds +2026-01-25 17:32:53,757 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:59,258 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:59,419 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:05,632 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:07,141 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:10,906 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:38,994 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:17,307 - INFO - Retrying request to /chat/completions in 0.451056 seconds +2026-01-25 17:34:47,442 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:51,701 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:56,299 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:08,957 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:44,633 - INFO - Retrying request to /chat/completions in 0.440974 seconds +2026-01-25 17:36:12,801 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:24,923 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:25,534 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:29,967 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:59,119 - INFO - Retrying request to /chat/completions in 0.452389 seconds +2026-01-25 17:37:20,423 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:37:24,953 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:37:44,633 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:08,048 - INFO - Retrying request to /chat/completions in 0.403657 seconds +2026-01-25 17:38:45,808 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:53,549 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:16,601 - INFO - Session round 4/5: 80 total, 59 sessions/hr +2026-01-25 17:39:16,605 - INFO - Retrying request to /chat/completions in 0.413989 seconds +2026-01-25 17:39:36,282 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:39,278 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:41,593 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:42,941 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:45,658 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:46,404 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:46,656 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:46,684 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:46,874 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:48,361 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:49,281 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:51,253 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:51,854 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:54,005 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:54,209 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:54,630 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:57,630 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:05,492 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:06,838 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:23,198 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:58,269 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:29,481 - INFO - Retrying request to /chat/completions in 0.493936 seconds +2026-01-25 17:44:48,890 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:49,136 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:53,558 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:04,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:08,806 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:13,776 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:14,132 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:14,613 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:14,997 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:15,469 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:19,208 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:19,330 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:20,748 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:21,521 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:22,916 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:24,595 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:27,458 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:27,988 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:29,146 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:29,421 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:47:53,546 - INFO - Retrying request to /chat/completions in 0.420826 seconds +2026-01-25 17:48:14,717 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:15,040 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:15,896 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:26,577 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:30,487 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:31,095 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:31,238 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:32,114 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:34,874 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:35,093 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:38,203 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:38,553 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:38,902 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:41,637 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:44,685 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:50,707 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:55,504 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:35,976 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:12,993 - INFO - Retrying request to /chat/completions in 0.484185 seconds +2026-01-25 17:51:29,368 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:31,418 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:35,813 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:41,708 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:56,845 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:51:59,749 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:15,342 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:18,520 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:20,747 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:30,124 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:53,875 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:16,080 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:30,129 - INFO - Retrying request to /chat/completions in 0.497456 seconds +2026-01-25 17:56:30,644 - INFO - Retrying request to /chat/completions in 0.935151 seconds +2026-01-25 17:58:17,168 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:43,175 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:00:49,060 - INFO - Retrying request to /chat/completions in 0.427525 seconds +2026-01-25 18:01:31,931 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:11,497 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:12,672 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:42,718 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:48,861 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:49,090 - INFO - Retrying request to /chat/completions in 0.456097 seconds +2026-01-25 18:04:37,643 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:42,728 - INFO - Retrying request to /chat/completions in 0.417728 seconds +2026-01-25 18:06:43,228 - INFO - Retrying request to /chat/completions in 0.769092 seconds +2026-01-25 18:10:46,099 - INFO - Retrying request to /chat/completions in 0.378243 seconds +2026-01-25 18:12:46,593 - INFO - Retrying request to /chat/completions in 0.822296 seconds +2026-01-25 18:14:22,523 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:15:08,469 - INFO - Retrying request to /chat/completions in 0.485664 seconds +2026-01-25 18:15:25,385 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:15:50,584 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:15:53,436 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:16:25,082 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +[2026-01-25T18:17:11.006] error: *** JOB 15640364 ON gpua005 CANCELLED AT 2026-01-25T18:17:11 DUE TO TIME LIMIT *** diff --git a/collaborativeagents/slurm/rag_vector_15640365.err b/collaborativeagents/slurm/rag_vector_15640365.err new file mode 100644 index 0000000..cb2777d --- /dev/null +++ b/collaborativeagents/slurm/rag_vector_15640365.err @@ -0,0 +1,659 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:08<00:26, 8.76s/it] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:10<00:09, 4.77s/it] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:17<00:05, 5.52s/it] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:22<00:00, 5.52s/it] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:22<00:00, 5.67s/it] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:03, 14.90it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:02, 15.68it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:02, 16.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:02, 16.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 11/51 [00:00<00:02, 18.23it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:00<00:01, 19.01it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:00<00:01, 19.75it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:01, 20.75it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:01<00:01, 21.38it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:01<00:01, 21.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:01<00:01, 19.86it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:01<00:00, 20.88it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:01<00:00, 21.81it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:01<00:00, 22.53it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:01<00:00, 23.19it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:02<00:00, 23.76it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:02<00:00, 23.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:02<00:00, 23.90it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:02<00:00, 20.76it/s] +[0;36m(EngineCore_DP0 pid=3400411)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:04, 7.53it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:01, 16.42it/s]
Capturing CUDA graphs (decode, FULL): 20%|██ | 7/35 [00:00<00:01, 19.40it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:01, 20.90it/s]
Capturing CUDA graphs (decode, FULL): 37%|███▋ | 13/35 [00:00<00:01, 21.91it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:00<00:00, 22.62it/s]
Capturing CUDA graphs (decode, FULL): 54%|█████▍ | 19/35 [00:00<00:00, 23.01it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:01<00:00, 23.31it/s]
Capturing CUDA graphs (decode, FULL): 71%|███████▏ | 25/35 [00:01<00:00, 23.57it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:01<00:00, 23.85it/s]
Capturing CUDA graphs (decode, FULL): 89%|████████▊ | 31/35 [00:01<00:00, 24.11it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:01<00:00, 24.12it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 22.45it/s] +[0;36m(APIServer pid=3399962)[0;0m INFO: Started server process [3399962] +[0;36m(APIServer pid=3399962)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=3399962)[0;0m INFO: Application startup complete. +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2026-01-25 16:44:27,527 - INFO - Loaded dataset: math-hard +2026-01-25 16:44:27,527 - INFO - Profile path configured: ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-25 16:44:27,550 - INFO - Loaded 200 profiles from ../data/complex_profiles_v2/profiles_200.jsonl +2026-01-25 16:44:27,551 - INFO - Running method: rag_vector +2026-01-25 16:44:27,552 - INFO - Running profiles 0 to 19 (20 remaining) +2026-01-25 16:44:27,552 - INFO - Using BATCH processing (async OpenAI user) for rag_vector +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:01<00:03, 1.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:02<00:02, 1.25s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:04<00:01, 1.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:04<00:00, 1.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:00<00:03, 1.03it/s]
Loading checkpoint shards: 40%|████ | 2/5 [00:01<00:02, 1.11it/s]
Loading checkpoint shards: 60%|██████ | 3/5 [00:02<00:01, 1.01it/s]
Loading checkpoint shards: 80%|████████ | 4/5 [00:03<00:00, 1.04it/s]
Loading checkpoint shards: 100%|██████████| 5/5 [00:03<00:00, 1.29it/s] +2026-01-25 16:46:33,420 - INFO - Batch: 20 profiles × 5 sessions, max 8 turns/session +2026-01-25 16:46:53,352 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:54,431 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:55,099 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:55,472 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:55,770 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:58,067 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:58,796 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:46:58,873 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:00,919 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:02,464 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:04,689 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:07,675 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:08,308 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:08,766 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:09,758 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:10,997 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:12,519 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:16,086 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:19,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:47:20,247 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2026-01-25 16:49:53,889 - INFO - Retrying request to /chat/completions in 0.431611 seconds +2026-01-25 16:50:19,052 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:20,345 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:21,832 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:28,044 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:28,069 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:30,740 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:31,459 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:34,351 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:35,558 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:37,035 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:37,040 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:38,007 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:46,691 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:48,958 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:49,437 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:53,900 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:56,558 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:50:59,477 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:02,171 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:05,693 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:51:30,656 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:16,552 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:20,513 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:52:43,757 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:53:04,635 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:53:26,686 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:53:58,915 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:54:07,870 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:54:18,415 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:54:34,551 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:54:48,270 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:00,168 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:15,848 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:31,660 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:55:43,673 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:00,322 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:26,583 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:33,774 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:47,336 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:56:58,962 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:09,127 - INFO - Retrying request to /chat/completions in 0.423172 seconds +2026-01-25 16:57:33,458 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:34,920 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:38,211 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:44,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:45,117 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:51,596 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:52,408 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:54,799 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:55,186 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:56,601 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:57:57,871 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:04,225 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:06,259 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:06,456 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:10,505 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:14,239 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:21,448 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:58:56,668 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:02,879 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:13,656 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:13,698 - INFO - Retrying request to /chat/completions in 0.402267 seconds +2026-01-25 16:59:31,806 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:53,097 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 16:59:57,819 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:06,774 - INFO - Retrying request to /chat/completions in 0.494621 seconds +2026-01-25 17:00:23,437 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:32,755 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:00:39,365 - INFO - Retrying request to /chat/completions in 0.408502 seconds +2026-01-25 17:00:55,143 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:01:05,972 - INFO - Retrying request to /chat/completions in 0.466327 seconds +2026-01-25 17:01:21,168 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:01:26,094 - INFO - Retrying request to /chat/completions in 0.489604 seconds +2026-01-25 17:01:33,513 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:01:43,187 - INFO - Retrying request to /chat/completions in 0.375130 seconds +2026-01-25 17:01:50,751 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:01:57,739 - INFO - Retrying request to /chat/completions in 0.375325 seconds +2026-01-25 17:02:10,000 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:15,119 - INFO - Retrying request to /chat/completions in 0.474180 seconds +2026-01-25 17:02:24,638 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:32,003 - INFO - Retrying request to /chat/completions in 0.489830 seconds +2026-01-25 17:02:37,430 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:02:44,806 - INFO - Retrying request to /chat/completions in 0.434947 seconds +2026-01-25 17:02:56,404 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:06,465 - INFO - Retrying request to /chat/completions in 0.403174 seconds +2026-01-25 17:03:13,287 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:19,983 - INFO - Retrying request to /chat/completions in 0.438961 seconds +2026-01-25 17:03:25,493 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:31,104 - INFO - Retrying request to /chat/completions in 0.489495 seconds +2026-01-25 17:03:40,011 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:03:50,270 - INFO - Retrying request to /chat/completions in 0.434883 seconds +2026-01-25 17:04:13,663 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:24,295 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:26,139 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 502 Bad Gateway" +2026-01-25 17:04:26,140 - INFO - Retrying request to /chat/completions in 0.436255 seconds +2026-01-25 17:04:34,478 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:40,798 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:48,269 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:50,716 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:52,567 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:53,758 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:04:59,710 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:00,330 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:16,314 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:19,553 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:05:50,726 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:06,983 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:49,614 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:06:49,662 - INFO - Retrying request to /chat/completions in 0.428723 seconds +2026-01-25 17:06:54,325 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:03,093 - INFO - Retrying request to /chat/completions in 0.379150 seconds +2026-01-25 17:07:11,184 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:13,984 - INFO - Retrying request to /chat/completions in 0.461548 seconds +2026-01-25 17:07:29,615 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:36,042 - INFO - Retrying request to /chat/completions in 0.462110 seconds +2026-01-25 17:07:46,117 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:07:48,597 - INFO - Retrying request to /chat/completions in 0.446762 seconds +2026-01-25 17:07:55,785 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:08:03,815 - INFO - Retrying request to /chat/completions in 0.455776 seconds +2026-01-25 17:08:28,631 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:08:36,264 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:08:42,613 - INFO - Retrying request to /chat/completions in 0.489013 seconds +2026-01-25 17:08:52,119 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:08:57,620 - INFO - Retrying request to /chat/completions in 0.454021 seconds +2026-01-25 17:09:04,652 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:13,602 - INFO - Retrying request to /chat/completions in 0.454350 seconds +2026-01-25 17:09:46,121 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:47,625 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:53,645 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:54,628 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:09:56,006 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:04,408 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:06,055 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:22,879 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:22,921 - INFO - Retrying request to /chat/completions in 0.429761 seconds +2026-01-25 17:10:26,980 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:31,364 - INFO - Retrying request to /chat/completions in 0.405246 seconds +2026-01-25 17:10:39,483 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:43,096 - INFO - Retrying request to /chat/completions in 0.440174 seconds +2026-01-25 17:10:50,605 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:10:57,750 - INFO - Retrying request to /chat/completions in 0.401265 seconds +2026-01-25 17:11:12,825 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:11:18,789 - INFO - Retrying request to /chat/completions in 0.419180 seconds +2026-01-25 17:11:33,449 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:11:41,792 - INFO - Retrying request to /chat/completions in 0.481257 seconds +2026-01-25 17:12:04,028 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:08,368 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:15,077 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:40,076 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:42,756 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:12:42,797 - INFO - Retrying request to /chat/completions in 0.424332 seconds +2026-01-25 17:12:54,602 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:00,455 - INFO - Retrying request to /chat/completions in 0.400305 seconds +2026-01-25 17:13:10,357 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:13,153 - INFO - Retrying request to /chat/completions in 0.494842 seconds +2026-01-25 17:13:20,182 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:25,818 - INFO - Retrying request to /chat/completions in 0.401322 seconds +2026-01-25 17:13:31,477 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:13:39,481 - INFO - Retrying request to /chat/completions in 0.406057 seconds +2026-01-25 17:13:59,598 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:01,407 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:06,500 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:20,527 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:20,568 - INFO - Retrying request to /chat/completions in 0.417564 seconds +2026-01-25 17:14:29,267 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:14:33,656 - INFO - Retrying request to /chat/completions in 0.388879 seconds +2026-01-25 17:14:56,373 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:15:05,031 - INFO - Retrying request to /chat/completions in 0.477432 seconds +2026-01-25 17:15:44,986 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:05,110 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:05,150 - INFO - Retrying request to /chat/completions in 0.432652 seconds +2026-01-25 17:16:13,641 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:17,962 - INFO - Retrying request to /chat/completions in 0.491335 seconds +2026-01-25 17:16:24,451 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:33,167 - INFO - Session round 1/5: 20 total, 37 sessions/hr +2026-01-25 17:16:33,169 - INFO - Retrying request to /chat/completions in 0.485946 seconds +2026-01-25 17:16:57,000 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:58,864 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:59,170 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:16:59,610 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:00,799 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:01,133 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:01,782 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:07,962 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:09,989 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:15,279 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:15,495 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:19,477 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:22,785 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:25,002 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:26,751 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:26,814 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:30,766 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:31,603 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:41,995 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:17:50,632 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:00,585 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:14,836 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:18,389 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:18:50,125 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:23,607 - INFO - Retrying request to /chat/completions in 0.441330 seconds +2026-01-25 17:21:42,752 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:55,540 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:56,845 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:21:59,095 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:06,986 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:07,428 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:09,354 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:14,719 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:14,932 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:18,785 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:19,290 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:24,424 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:26,825 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:26,893 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:27,480 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:27,679 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:28,681 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:36,239 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:38,562 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:22:42,497 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:15,879 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:27,558 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:28,587 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:30,431 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:31,539 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:32,487 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:32,934 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:53,576 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:53,911 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:23:53,952 - INFO - Retrying request to /chat/completions in 0.387449 seconds +2026-01-25 17:23:58,728 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:24:06,098 - INFO - Retrying request to /chat/completions in 0.417363 seconds +2026-01-25 17:24:22,739 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:24:27,247 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:24:30,742 - INFO - Retrying request to /chat/completions in 0.427253 seconds +2026-01-25 17:24:35,044 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:24:42,806 - INFO - Retrying request to /chat/completions in 0.430509 seconds +2026-01-25 17:24:47,621 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:24:56,147 - INFO - Retrying request to /chat/completions in 0.406768 seconds +2026-01-25 17:25:08,158 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:25:15,280 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:25:37,132 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:25:44,302 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:01,934 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:16,742 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:20,927 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:29,524 - INFO - Retrying request to /chat/completions in 0.481632 seconds +2026-01-25 17:26:33,297 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:26:42,580 - INFO - Retrying request to /chat/completions in 0.379269 seconds +2026-01-25 17:26:47,544 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:27:01,303 - INFO - Retrying request to /chat/completions in 0.468233 seconds +2026-01-25 17:27:06,015 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:27:11,650 - INFO - Retrying request to /chat/completions in 0.446647 seconds +2026-01-25 17:27:15,210 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:27:19,655 - INFO - Retrying request to /chat/completions in 0.497708 seconds +2026-01-25 17:27:23,777 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:27:31,605 - INFO - Retrying request to /chat/completions in 0.479417 seconds +2026-01-25 17:27:35,435 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:27:42,838 - INFO - Retrying request to /chat/completions in 0.461569 seconds +2026-01-25 17:27:51,849 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:28:05,557 - INFO - Retrying request to /chat/completions in 0.498239 seconds +2026-01-25 17:28:13,080 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:28:21,705 - INFO - Retrying request to /chat/completions in 0.385335 seconds +2026-01-25 17:28:25,437 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:28:37,198 - INFO - Retrying request to /chat/completions in 0.391341 seconds +2026-01-25 17:28:59,627 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:08,328 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:10,089 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:12,861 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:14,134 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:14,508 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:18,761 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:18,849 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:21,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:33,641 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:38,843 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:39,412 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:44,783 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:44,914 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:46,694 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:46,889 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:29:47,342 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:14,719 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:21,514 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:22,403 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:30:41,585 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:13,657 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:16,145 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:16,211 - INFO - Retrying request to /chat/completions in 0.376739 seconds +2026-01-25 17:31:22,696 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:32,224 - INFO - Retrying request to /chat/completions in 0.481756 seconds +2026-01-25 17:31:37,232 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:39,939 - INFO - Retrying request to /chat/completions in 0.464822 seconds +2026-01-25 17:31:43,685 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:31:50,154 - INFO - Retrying request to /chat/completions in 0.497032 seconds +2026-01-25 17:31:55,994 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:02,164 - INFO - Retrying request to /chat/completions in 0.420697 seconds +2026-01-25 17:32:08,186 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:12,775 - INFO - Retrying request to /chat/completions in 0.418960 seconds +2026-01-25 17:32:17,372 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:27,814 - INFO - Retrying request to /chat/completions in 0.416794 seconds +2026-01-25 17:32:42,720 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:32:53,751 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:01,549 - INFO - Retrying request to /chat/completions in 0.436067 seconds +2026-01-25 17:33:06,593 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:13,684 - INFO - Retrying request to /chat/completions in 0.433770 seconds +2026-01-25 17:33:20,885 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:33:33,652 - INFO - Retrying request to /chat/completions in 0.434689 seconds +2026-01-25 17:33:50,177 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:13,554 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:18,034 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:26,346 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:33,023 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:37,787 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:45,897 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:49,121 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:34:52,933 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:37,205 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:35:58,100 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:12,632 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:36:49,131 - INFO - Retrying request to /chat/completions in 0.396448 seconds +2026-01-25 17:38:12,564 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:12,630 - INFO - Retrying request to /chat/completions in 0.419012 seconds +2026-01-25 17:38:16,918 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:25,849 - INFO - Retrying request to /chat/completions in 0.426402 seconds +2026-01-25 17:38:30,802 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:37,629 - INFO - Retrying request to /chat/completions in 0.447026 seconds +2026-01-25 17:38:42,352 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:38:44,326 - INFO - Retrying request to /chat/completions in 0.413236 seconds +2026-01-25 17:38:50,341 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:00,284 - INFO - Retrying request to /chat/completions in 0.451115 seconds +2026-01-25 17:39:07,431 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:16,904 - INFO - Retrying request to /chat/completions in 0.450390 seconds +2026-01-25 17:39:38,144 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:48,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:39:54,096 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:04,366 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:40:43,687 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:47,492 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:41:47,557 - INFO - Retrying request to /chat/completions in 0.494924 seconds +2026-01-25 17:42:01,193 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:42:06,412 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:42:16,774 - INFO - Retrying request to /chat/completions in 0.460608 seconds +2026-01-25 17:42:21,532 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:42:27,428 - INFO - Retrying request to /chat/completions in 0.400003 seconds +2026-01-25 17:42:34,009 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:42:35,894 - INFO - Retrying request to /chat/completions in 0.386291 seconds +2026-01-25 17:42:41,838 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:42:51,314 - INFO - Retrying request to /chat/completions in 0.416458 seconds +2026-01-25 17:42:54,416 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:43:04,527 - INFO - Retrying request to /chat/completions in 0.456470 seconds +2026-01-25 17:43:34,255 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:43:37,278 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:43:40,968 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:00,736 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:05,123 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:05,190 - INFO - Retrying request to /chat/completions in 0.455121 seconds +2026-01-25 17:44:09,678 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:21,444 - INFO - Retrying request to /chat/completions in 0.489223 seconds +2026-01-25 17:44:24,727 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:31,398 - INFO - Retrying request to /chat/completions in 0.447302 seconds +2026-01-25 17:44:35,655 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:37,398 - INFO - Retrying request to /chat/completions in 0.432397 seconds +2026-01-25 17:44:42,206 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:44:53,588 - INFO - Retrying request to /chat/completions in 0.410011 seconds +2026-01-25 17:44:57,328 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:07,532 - INFO - Retrying request to /chat/completions in 0.438445 seconds +2026-01-25 17:45:30,715 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:46,678 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:46,686 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:45:59,866 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:07,608 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:07,649 - INFO - Retrying request to /chat/completions in 0.487105 seconds +2026-01-25 17:46:13,228 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:20,359 - INFO - Retrying request to /chat/completions in 0.468930 seconds +2026-01-25 17:46:23,958 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:36,511 - INFO - Retrying request to /chat/completions in 0.394460 seconds +2026-01-25 17:46:40,672 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:46:53,349 - INFO - Retrying request to /chat/completions in 0.381270 seconds +2026-01-25 17:47:20,885 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:47:35,648 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:47:45,373 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:09,534 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:09,574 - INFO - Retrying request to /chat/completions in 0.397584 seconds +2026-01-25 17:48:13,711 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:20,469 - INFO - Retrying request to /chat/completions in 0.451293 seconds +2026-01-25 17:48:27,382 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:39,177 - INFO - Retrying request to /chat/completions in 0.445611 seconds +2026-01-25 17:48:41,990 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:48:54,163 - INFO - Session round 2/5: 40 total, 37 sessions/hr +2026-01-25 17:48:54,166 - INFO - Retrying request to /chat/completions in 0.405459 seconds +2026-01-25 17:49:08,413 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:10,823 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:15,020 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:15,483 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:15,805 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:16,157 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:17,506 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:17,629 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:18,139 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:21,077 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:21,288 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:21,614 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:22,373 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:22,396 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:22,509 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:23,876 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:24,614 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:24,808 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:29,324 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:49:29,384 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:16,595 - INFO - Retrying request to /chat/completions in 0.375700 seconds +2026-01-25 17:52:31,824 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:41,658 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:48,979 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:49,698 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:51,018 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:51,309 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:53,635 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:53,962 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:55,517 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:57,204 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:57,970 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:59,659 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:52:59,991 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:03,472 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:04,616 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:05,774 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:10,326 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:13,739 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:23,719 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:25,547 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:25,606 - INFO - Retrying request to /chat/completions in 0.394612 seconds +2026-01-25 17:53:32,439 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:53:42,523 - INFO - Retrying request to /chat/completions in 0.490730 seconds +2026-01-25 17:53:53,687 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:01,296 - INFO - Retrying request to /chat/completions in 0.396549 seconds +2026-01-25 17:54:06,442 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:17,695 - INFO - Retrying request to /chat/completions in 0.485713 seconds +2026-01-25 17:54:24,813 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:37,355 - INFO - Retrying request to /chat/completions in 0.384747 seconds +2026-01-25 17:54:45,435 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:54:56,273 - INFO - Retrying request to /chat/completions in 0.383095 seconds +2026-01-25 17:55:01,761 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:09,535 - INFO - Retrying request to /chat/completions in 0.483591 seconds +2026-01-25 17:55:17,052 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:24,734 - INFO - Retrying request to /chat/completions in 0.496542 seconds +2026-01-25 17:55:28,909 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:36,287 - INFO - Retrying request to /chat/completions in 0.395408 seconds +2026-01-25 17:55:39,919 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:55:44,324 - INFO - Retrying request to /chat/completions in 0.434599 seconds +2026-01-25 17:55:55,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:56:02,080 - INFO - Retrying request to /chat/completions in 0.476309 seconds +2026-01-25 17:56:10,965 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:56:18,792 - INFO - Retrying request to /chat/completions in 0.389062 seconds +2026-01-25 17:56:26,607 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:56:41,112 - INFO - Retrying request to /chat/completions in 0.408340 seconds +2026-01-25 17:56:52,754 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:57:01,723 - INFO - Retrying request to /chat/completions in 0.466930 seconds +2026-01-25 17:57:05,928 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:57:16,234 - INFO - Retrying request to /chat/completions in 0.469474 seconds +2026-01-25 17:57:20,966 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:57:29,059 - INFO - Retrying request to /chat/completions in 0.482807 seconds +2026-01-25 17:57:51,500 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:57:52,343 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:57:56,836 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:57:58,529 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:02,103 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:02,448 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:02,451 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:03,109 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:06,494 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:13,281 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:15,778 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:18,386 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:21,050 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:29,406 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:44,160 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:58:44,202 - INFO - Retrying request to /chat/completions in 0.431804 seconds +2026-01-25 17:58:50,624 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:02,026 - INFO - Retrying request to /chat/completions in 0.421486 seconds +2026-01-25 17:59:09,691 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:21,551 - INFO - Retrying request to /chat/completions in 0.479624 seconds +2026-01-25 17:59:26,503 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:33,556 - INFO - Retrying request to /chat/completions in 0.478310 seconds +2026-01-25 17:59:43,515 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 17:59:51,296 - INFO - Retrying request to /chat/completions in 0.376088 seconds +2026-01-25 18:00:05,212 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:00:11,413 - INFO - Retrying request to /chat/completions in 0.411704 seconds +2026-01-25 18:00:20,148 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:00:27,431 - INFO - Retrying request to /chat/completions in 0.485240 seconds +2026-01-25 18:00:33,844 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:00:45,541 - INFO - Retrying request to /chat/completions in 0.395221 seconds +2026-01-25 18:00:49,595 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:00:58,755 - INFO - Retrying request to /chat/completions in 0.482343 seconds +2026-01-25 18:01:15,046 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:01:24,985 - INFO - Retrying request to /chat/completions in 0.493180 seconds +2026-01-25 18:01:36,175 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:01:48,186 - INFO - Retrying request to /chat/completions in 0.430112 seconds +2026-01-25 18:02:31,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:46,124 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:02:52,172 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:01,039 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:04,711 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:11,672 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:26,850 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:32,704 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:41,257 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:03:48,235 - INFO - Retrying request to /chat/completions in 0.492124 seconds +2026-01-25 18:04:52,916 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:04:52,957 - INFO - Retrying request to /chat/completions in 0.393496 seconds +2026-01-25 18:04:59,315 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:05:09,942 - INFO - Retrying request to /chat/completions in 0.437104 seconds +2026-01-25 18:05:18,058 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:05:27,688 - INFO - Retrying request to /chat/completions in 0.474594 seconds +2026-01-25 18:05:35,494 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:05:40,891 - INFO - Retrying request to /chat/completions in 0.484824 seconds +2026-01-25 18:05:47,222 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:05:57,550 - INFO - Retrying request to /chat/completions in 0.400724 seconds +2026-01-25 18:06:09,119 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:06:14,309 - INFO - Retrying request to /chat/completions in 0.428613 seconds +2026-01-25 18:06:19,645 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:06:29,938 - INFO - Retrying request to /chat/completions in 0.439041 seconds +2026-01-25 18:06:35,821 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:06:47,099 - INFO - Retrying request to /chat/completions in 0.378707 seconds +2026-01-25 18:07:17,343 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:35,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:44,048 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:46,288 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:50,703 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:07:55,831 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:08:47,140 - INFO - Retrying request to /chat/completions in 0.376425 seconds +2026-01-25 18:09:48,953 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:09:48,996 - INFO - Retrying request to /chat/completions in 0.401510 seconds +2026-01-25 18:10:01,057 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:10:13,367 - INFO - Retrying request to /chat/completions in 0.452543 seconds +2026-01-25 18:10:23,313 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:10:29,506 - INFO - Retrying request to /chat/completions in 0.456059 seconds +2026-01-25 18:10:34,648 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:10:56,384 - INFO - Retrying request to /chat/completions in 0.423241 seconds +2026-01-25 18:11:08,182 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:11:17,057 - INFO - Retrying request to /chat/completions in 0.387066 seconds +2026-01-25 18:11:22,865 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:11:32,075 - INFO - Retrying request to /chat/completions in 0.450292 seconds +2026-01-25 18:11:36,487 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:11:46,486 - INFO - Retrying request to /chat/completions in 0.446748 seconds +2026-01-25 18:12:11,352 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:19,046 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:20,537 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:21,672 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:22,945 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:28,935 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:28,978 - INFO - Retrying request to /chat/completions in 0.404645 seconds +2026-01-25 18:12:36,324 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:47,355 - INFO - Retrying request to /chat/completions in 0.385300 seconds +2026-01-25 18:12:51,437 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:12:59,467 - INFO - Retrying request to /chat/completions in 0.472282 seconds +2026-01-25 18:13:05,236 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:13:17,804 - INFO - Retrying request to /chat/completions in 0.400540 seconds +2026-01-25 18:13:27,259 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:13:37,534 - INFO - Retrying request to /chat/completions in 0.410282 seconds +2026-01-25 18:13:42,043 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:13:52,134 - INFO - Retrying request to /chat/completions in 0.400339 seconds +2026-01-25 18:14:34,854 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:14:36,311 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:14:38,639 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:14:42,380 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:14:59,719 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:14:59,762 - INFO - Retrying request to /chat/completions in 0.453975 seconds +2026-01-25 18:15:07,860 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:15:20,608 - INFO - Retrying request to /chat/completions in 0.478967 seconds +2026-01-25 18:15:24,948 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:15:35,230 - INFO - Retrying request to /chat/completions in 0.474441 seconds +2026-01-25 18:15:44,294 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:15:58,584 - INFO - Retrying request to /chat/completions in 0.457305 seconds +2026-01-25 18:16:03,282 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:16:15,075 - INFO - Retrying request to /chat/completions in 0.457749 seconds +2026-01-25 18:16:40,579 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:17:02,606 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:17:14,884 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:17:26,506 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:17:26,550 - INFO - Retrying request to /chat/completions in 0.414474 seconds +2026-01-25 18:17:33,006 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:17:44,484 - INFO - Retrying request to /chat/completions in 0.436180 seconds +2026-01-25 18:17:50,546 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:18:01,972 - INFO - Retrying request to /chat/completions in 0.421992 seconds +2026-01-25 18:18:08,775 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +2026-01-25 18:18:19,604 - INFO - Retrying request to /chat/completions in 0.487531 seconds +2026-01-25 18:18:24,153 - INFO - HTTP Request: POST https://api.openai.com/v1/chat/completions "HTTP/1.1 200 OK" +OSError: [Errno 122] Disk quota exceeded + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 1184, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 1169, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 918, in run_all + results = self.run_method(method) + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 536, in run_method + return self._run_method_batch( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 893, in _run_method_batch + with open(results_file, "w") as f: +OSError: [Errno 5] Input/output error diff --git a/collaborativeagents/slurm/rerun_reflection.sbatch b/collaborativeagents/slurm/rerun_reflection.sbatch new file mode 100644 index 0000000..aac2ed3 --- /dev/null +++ b/collaborativeagents/slurm/rerun_reflection.sbatch @@ -0,0 +1,83 @@ +#!/bin/bash +#SBATCH --job-name=refl_fix +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8-interactive +#SBATCH --gres=gpu:4 +#SBATCH --time=01:00:00 +#SBATCH --mem=200G +#SBATCH --cpus-per-task=32 +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source ~/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model:$PYTHONPATH + +PROFILE_PATH="collaborativeagents/data/complex_profiles_v2/profiles_200.jsonl" +AGENT_MODEL="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +USER_MODEL="meta-llama/Llama-3.1-70B-Instruct" + +echo "=== Starting vLLM servers ===" +date + +# User simulator on GPUs 0,1 (70B) +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $USER_MODEL \ + --port 8004 --tensor-parallel-size 2 --gpu-memory-utilization 0.90 \ + --max-model-len 16384 --dtype bfloat16 --download-dir $HF_HOME & + +# Agent on GPUs 2,3 (8B) +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $AGENT_MODEL \ + --port 8003 --tensor-parallel-size 2 --gpu-memory-utilization 0.90 \ + --max-model-len 16384 --dtype bfloat16 & + +# Wait for servers +echo "Waiting for vLLM servers..." +for i in {1..200}; do + if curl -s http://localhost:8004/health > /dev/null 2>&1; then + echo "User simulator (8004) ready after $((i*5)) seconds" + break + fi + sleep 5 +done +for i in {1..60}; do + if curl -s http://localhost:8003/health > /dev/null 2>&1; then + echo "Agent (8003) ready after $((i*5)) seconds" + break + fi + sleep 5 +done +echo "Both vLLM servers ready" +sleep 10 + +# Run profile 1 (user_14b429db - had empty response bug) +echo "=== Running profile 1 (user_14b429db) ===" +python collaborativeagents/scripts/run_experiments.py \ + --methods reflection \ + --n-profiles 5 \ + --n-sessions 15 \ + --start-profile 1 --end-profile 2 \ + --output-dir results/reflection_rerun \ + --profile-path $PROFILE_PATH \ + --datasets math-hard \ + --use-vllm --parallel-profiles 1 --no-batch-processing + +# Run profile 4 (user_a0a3ed44 - was missing) +echo "=== Running profile 4 (user_a0a3ed44) ===" +python collaborativeagents/scripts/run_experiments.py \ + --methods reflection \ + --n-profiles 5 \ + --n-sessions 15 \ + --start-profile 4 --end-profile 5 \ + --output-dir results/reflection_rerun \ + --profile-path $PROFILE_PATH \ + --datasets math-hard \ + --use-vllm --parallel-profiles 1 --no-batch-processing + +pkill -f "vllm.entrypoints" 2>/dev/null || true +echo "Done!" diff --git a/collaborativeagents/slurm/run_all_gpt_user_test.sh b/collaborativeagents/slurm/run_all_gpt_user_test.sh new file mode 100644 index 0000000..26a1677 --- /dev/null +++ b/collaborativeagents/slurm/run_all_gpt_user_test.sh @@ -0,0 +1,67 @@ +#!/bin/bash +#SBATCH --job-name=gpt_user_test +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=4:00:00 +#SBATCH --output=gpt_user_test_%j.out +#SBATCH --error=gpt_user_test_%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +# Load OpenAI API key +set -a +source .env +set +a + +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# GPU 0: vLLM 8B agent, GPU 1: adapter models (embedding/extractor/reranker) +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +# Wait for server +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM 8B agent server ready." + +cd collaborativeagents/scripts + +# Large scale test: 50 profiles in parallel, 3 sessions each, all 6 methods +python run_experiments.py \ + --methods vanilla,contextual,reflection,all_memory,rag,rag_vector \ + --datasets math-hard \ + --n-profiles 20 \ + --n-sessions 5 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5 \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 20 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/gpt_user_scale_test_$(date +%Y%m%d_%H%M%S) + +echo "All methods test complete!" + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_all_memory.sh b/collaborativeagents/slurm/run_all_memory.sh new file mode 100755 index 0000000..b32cde6 --- /dev/null +++ b/collaborativeagents/slurm/run_all_memory.sh @@ -0,0 +1,57 @@ +#!/bin/bash +#SBATCH --job-name=all_memory +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=2:00:00 +#SBATCH --output=all_memory_%j.out +#SBATCH --error=all_memory_%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a; source .env; set +a +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods all_memory \ + --datasets math-hard \ + --n-profiles 20 \ + --n-sessions 5 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5 \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 20 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/gpt_user_scale_all_memory_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_all_memory_v2.sh b/collaborativeagents/slurm/run_all_memory_v2.sh new file mode 100755 index 0000000..4caa29a --- /dev/null +++ b/collaborativeagents/slurm/run_all_memory_v2.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=all_mem_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/all_memory_v2_%j.out +#SBATCH --error=logs/all_memory_v2_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# all_memory with FIXED memory paths +python run_experiments.py \ + --methods all_memory \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/all_memory_v2_$(date +%Y%m%d_%H%M%S) diff --git a/collaborativeagents/slurm/run_contextual.sh b/collaborativeagents/slurm/run_contextual.sh new file mode 100755 index 0000000..c5ac1b0 --- /dev/null +++ b/collaborativeagents/slurm/run_contextual.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#SBATCH --job-name=contextual +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/contextual_%j.out +#SBATCH --error=logs/contextual_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods contextual \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/contextual_$(date +%Y%m%d_%H%M%S) diff --git a/collaborativeagents/slurm/run_contextual_p0_9.sh b/collaborativeagents/slurm/run_contextual_p0_9.sh new file mode 100755 index 0000000..b78afd0 --- /dev/null +++ b/collaborativeagents/slurm/run_contextual_p0_9.sh @@ -0,0 +1,36 @@ +#!/bin/bash +#SBATCH --job-name=ctx_p0_9 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/contextual_p0_9_%j.out +#SBATCH --error=logs/contextual_p0_9_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# Run profiles 0-9 (10 profiles, ~40 hours) +python run_experiments.py \ + --methods contextual \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/contextual_20251227_020146 \ + --start-profile 0 \ + --end-profile 10 diff --git a/collaborativeagents/slurm/run_contextual_p10_19.sh b/collaborativeagents/slurm/run_contextual_p10_19.sh new file mode 100755 index 0000000..64bd5e2 --- /dev/null +++ b/collaborativeagents/slurm/run_contextual_p10_19.sh @@ -0,0 +1,36 @@ +#!/bin/bash +#SBATCH --job-name=ctx_p10_19 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/contextual_p10_19_%j.out +#SBATCH --error=logs/contextual_p10_19_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# Run profiles 10-19 (10 profiles, ~40 hours) +python run_experiments.py \ + --methods contextual \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/contextual_20251227_020146 \ + --start-profile 10 \ + --end-profile 20 diff --git a/collaborativeagents/slurm/run_contextual_p20_29.sh b/collaborativeagents/slurm/run_contextual_p20_29.sh new file mode 100755 index 0000000..fb94734 --- /dev/null +++ b/collaborativeagents/slurm/run_contextual_p20_29.sh @@ -0,0 +1,36 @@ +#!/bin/bash +#SBATCH --job-name=ctx_p20_29 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/contextual_p20_29_%j.out +#SBATCH --error=logs/contextual_p20_29_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# Run profiles 20-29 (10 profiles, ~40 hours) +python run_experiments.py \ + --methods contextual \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/contextual_20251227_020146 \ + --start-profile 20 \ + --end-profile 30 diff --git a/collaborativeagents/slurm/run_contextual_resume.sh b/collaborativeagents/slurm/run_contextual_resume.sh new file mode 100755 index 0000000..2dec969 --- /dev/null +++ b/collaborativeagents/slurm/run_contextual_resume.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctx_resume +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/contextual_resume_%j.out +#SBATCH --error=logs/contextual_resume_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# Resume from the EXISTING output directory (uses checkpoint) +python run_experiments.py \ + --methods contextual \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/contextual_20251227_020146 diff --git a/collaborativeagents/slurm/run_contextual_v2.sh b/collaborativeagents/slurm/run_contextual_v2.sh new file mode 100755 index 0000000..0cf69e0 --- /dev/null +++ b/collaborativeagents/slurm/run_contextual_v2.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#SBATCH --job-name=ctx_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=2:00:00 +#SBATCH --output=ctx_v2_%j.out +#SBATCH --error=ctx_v2_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.90 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods contextual \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/contextual_v2_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_experiments.sh b/collaborativeagents/slurm/run_experiments.sh new file mode 100644 index 0000000..e254202 --- /dev/null +++ b/collaborativeagents/slurm/run_experiments.sh @@ -0,0 +1,66 @@ +#!/bin/bash +#SBATCH --job-name=run_expts +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:8 +#SBATCH --mem=400G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/run_expts_%j.out +#SBATCH --error=logs/run_expts_%j.err + +# Run experiments with models loaded locally +# This job needs 8 GPUs: +# - 4 GPUs for 70B judge model +# - 2 GPUs for PersonalizedLLM (embedder, reranker, extractor, main LLM) +# - Reserve for headroom + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs and results directories +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/results + +echo "Starting experiments at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +# Activate environment +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Check GPU availability +nvidia-smi + +# Add project to path +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Run experiments +cd collaborativeagents/scripts + +# Quick test first (2 profiles, 2 sessions) +echo "Running quick test..." +python run_experiments.py \ + --methods rag_vector \ + --datasets math-500 \ + --n-profiles 2 \ + --n-sessions 2 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/test_$(date +%Y%m%d_%H%M%S) + +# Full run (uncomment when ready) +# echo "Running full experiments..." +# python run_experiments.py \ +# --methods vanilla,all_memory,rag,rag_vector \ +# --datasets math-500,gpqa,aime \ +# --n-profiles 100 \ +# --n-sessions 20 \ +# --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ +# --output-dir ../results/full_$(date +%Y%m%d_%H%M%S) + +echo "Experiments completed at $(date)" diff --git a/collaborativeagents/slurm/run_experiments_a100.sh b/collaborativeagents/slurm/run_experiments_a100.sh new file mode 100644 index 0000000..aa2e658 --- /dev/null +++ b/collaborativeagents/slurm/run_experiments_a100.sh @@ -0,0 +1,58 @@ +#!/bin/bash +#SBATCH --job-name=run_expts_a100 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/run_expts_a100_%j.out +#SBATCH --error=logs/run_expts_a100_%j.err + +# Run experiments on 4x A100 80GB +# - 70B judge model with TP=4 (~140GB) +# - 8B PersonalizedLLM models (~16GB shared) + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs and results directories +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/results + +echo "Starting experiments at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +# Activate environment +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Check GPU availability +nvidia-smi + +# Redirect HF cache to project space (avoid home quota issues) +export HF_HOME=/projects/bfqt/users/yurenh2/.cache/huggingface +mkdir -p $HF_HOME + +# Add project to path +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Run experiments +cd collaborativeagents/scripts + +# Full benchmark run +echo "Running full experiments..." +python run_experiments.py \ + --methods vanilla,all_memory,rag,rag_vector \ + --datasets math-500 \ + --n-profiles 20 \ + --n-sessions 5 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/full_$(date +%Y%m%d_%H%M%S) + +echo "Experiments completed at $(date)" diff --git a/collaborativeagents/slurm/run_experiments_collab_baselines.sh b/collaborativeagents/slurm/run_experiments_collab_baselines.sh new file mode 100755 index 0000000..bbf53fc --- /dev/null +++ b/collaborativeagents/slurm/run_experiments_collab_baselines.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#SBATCH --job-name=run_collab_baselines +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/run_collab_baselines_%j.out +#SBATCH --error=logs/run_collab_baselines_%j.err + +# Run CollaborativeAgents baselines on 4x A100 80GB +# - contextual: Full history in context (summarize on overflow) +# - reflection: CollaborativeAgents' agent_notes approach +# - reflection_grpo: Reflection + GRPO training (with_proper_scaffolding) + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs and results directories +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/results + +echo "Starting CollaborativeAgents baselines at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +# Activate environment +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Check GPU availability +nvidia-smi + +# Redirect HF cache to project space (avoid home quota issues) +export HF_HOME=/projects/bfqt/users/yurenh2/.cache/huggingface +mkdir -p $HF_HOME + +# Add project to path +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Run experiments +cd collaborativeagents/scripts + +# Run the 3 CollaborativeAgents baselines +echo "Running contextual, reflection, reflection_grpo baselines..." +python run_experiments.py \ + --methods contextual,reflection,reflection_grpo \ + --datasets math-500 \ + --n-profiles 20 \ + --n-sessions 5 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/collab_baselines_$(date +%Y%m%d_%H%M%S) + +echo "CollaborativeAgents baselines completed at $(date)" diff --git a/collaborativeagents/slurm/run_experiments_multiturn.sh b/collaborativeagents/slurm/run_experiments_multiturn.sh new file mode 100755 index 0000000..ca5c04c --- /dev/null +++ b/collaborativeagents/slurm/run_experiments_multiturn.sh @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH --job-name=run_multiturn +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/run_multiturn_%j.out +#SBATCH --error=logs/run_multiturn_%j.err + +# Run FIXED experiment with proper multi-turn conversation and user simulation +# This uses LocalUserAgent for user simulation and proper metrics extraction + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs and results directories +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/results + +echo "Starting MULTI-TURN experiment at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +# Activate environment +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Check GPU availability +nvidia-smi + +# Redirect HF cache to project space (avoid home quota issues) +export HF_HOME=/projects/bfqt/users/yurenh2/.cache/huggingface +mkdir -p $HF_HOME + +# Add project to path +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Run experiments +cd collaborativeagents/scripts + +# INITIAL TEST: Run with reduced parameters to validate the fix +# - 5 profiles (instead of 20) +# - 3 sessions per profile (instead of 5) +# - All 7 methods +echo "Running MULTI-TURN experiments with user simulation..." +python run_experiments.py \ + --methods vanilla,all_memory,rag,rag_vector,contextual,reflection,reflection_grpo \ + --datasets math-500 \ + --n-profiles 5 \ + --n-sessions 3 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/multiturn_test_$(date +%Y%m%d_%H%M%S) + +echo "Multi-turn experiment completed at $(date)" diff --git a/collaborativeagents/slurm/run_full_experiment.sh b/collaborativeagents/slurm/run_full_experiment.sh new file mode 100755 index 0000000..bebe58c --- /dev/null +++ b/collaborativeagents/slurm/run_full_experiment.sh @@ -0,0 +1,58 @@ +#!/bin/bash +#SBATCH --job-name=full_exp +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/full_exp_%j.out +#SBATCH --error=logs/full_exp_%j.err + +# Full scale experiment with 70B user model +# ORIGINAL CONFIG - DO NOT CHANGE + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/results + +echo "Starting FULL SCALE experiment at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +nvidia-smi + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +mkdir -p $HF_HOME + +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Fix for nvlink errors +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# ORIGINAL FULL SCALE CONFIG: +# - 30 profiles +# - 20 sessions per profile +# - 4 challenging datasets: gpqa, aime, math-hard, humaneval +# - All 7 methods +echo "Running FULL SCALE experiments with 70B user model..." +python run_experiments.py \ + --methods vanilla,all_memory,rag,rag_vector,contextual,reflection,reflection_grpo \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/full_experiment_$(date +%Y%m%d_%H%M%S) + +echo "Full experiment completed at $(date)" diff --git a/collaborativeagents/slurm/run_full_experiment_v2.sh b/collaborativeagents/slurm/run_full_experiment_v2.sh new file mode 100755 index 0000000..ab77478 --- /dev/null +++ b/collaborativeagents/slurm/run_full_experiment_v2.sh @@ -0,0 +1,61 @@ +#!/bin/bash +#SBATCH --job-name=full_exp_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/full_exp_v2_%j.out +#SBATCH --error=logs/full_exp_v2_%j.err + +# Full scale experiment v2 - with fixes: +# 1. Use stable datasets (math-500, humaneval) - avoid problematic ones +# 2. Reduced scale first (20 profiles, 15 sessions) to verify stability +# 3. Clear CUDA cache between methods + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +mkdir -p collaborativeagents/slurm/logs +mkdir -p collaborativeagents/results + +echo "Starting FULL SCALE v2 experiment at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +nvidia-smi + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +mkdir -p $HF_HOME + +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Disable peer-to-peer memory access to avoid nvlink errors +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts + +# FULL SCALE v2: +# - 20 profiles (reduced from 30 for stability) +# - 15 sessions per profile (enough to show learning) +# - 2 stable datasets: math-500, humaneval +# - All 7 methods +echo "Running FULL SCALE v2 experiments..." +python run_experiments.py \ + --methods vanilla,all_memory,rag,rag_vector,contextual,reflection,reflection_grpo \ + --datasets math-500,humaneval \ + --n-profiles 20 \ + --n-sessions 15 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/full_experiment_v2_$(date +%Y%m%d_%H%M%S) + +echo "Full experiment v2 completed at $(date)" diff --git a/collaborativeagents/slurm/run_grpo_test.sh b/collaborativeagents/slurm/run_grpo_test.sh new file mode 100644 index 0000000..199c540 --- /dev/null +++ b/collaborativeagents/slurm/run_grpo_test.sh @@ -0,0 +1,101 @@ +#!/bin/bash +#SBATCH --job-name=grpo_test +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:4 +#SBATCH --mem=200G +#SBATCH --time=1:00:00 +#SBATCH --output=grpo_test_%j.out +#SBATCH --error=grpo_test_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Install required packages (ensure they're in the conda env) +echo "Installing required packages..." +pip install --quiet json-repair tenacity + +# Test: Verify imports work +echo "Testing imports..." +python3 -c "from json_repair import repair_json; from tenacity import retry; print('Imports OK')" + +# Start judge model (70B) on GPUs 2,3 +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" + +echo "Starting judge model..." +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 8192 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +# Wait for server +for i in $(seq 1 60); do + curl -s http://localhost:8004/health > /dev/null 2>&1 && break + sleep 3 +done +echo "Judge model ready" + +# Run GRPO with minimal steps for testing +echo "Starting GRPO test (10 steps only)..." +cd collaborativeagents/training/grpo_verl + +python -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + data.train_files=${PWD}/data/session_level_reflection_grpo_train.parquet \ + data.val_files=${PWD}/data/session_level_reflection_grpo_train.parquet \ + data.train_batch_size=8 \ + data.max_prompt_length=2048 \ + data.max_response_length=1024 \ + data.filter_overlong_prompts=True \ + data.truncation=error \ + data.prompt_key=prompt \ + data.reward_fn_key=data_source \ + actor_rollout_ref.model.path=/work/nvme/bfqt/yurenh2/sft_checkpoints/checkpoint-200 \ + actor_rollout_ref.actor.optim.lr=1e-6 \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.actor.ppo_mini_batch_size=4 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.actor.use_kl_loss=True \ + actor_rollout_ref.actor.kl_loss_coef=0.003 \ + actor_rollout_ref.actor.kl_loss_type=low_var_kl \ + actor_rollout_ref.actor.entropy_coeff=0 \ + actor_rollout_ref.model.enable_gradient_checkpointing=True \ + actor_rollout_ref.actor.fsdp_config.model_dtype=bfloat16 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \ + actor_rollout_ref.ref.fsdp_config.model_dtype=bfloat16 \ + actor_rollout_ref.ref.fsdp_config.param_offload=True \ + actor_rollout_ref.rollout.temperature=0.9 \ + actor_rollout_ref.rollout.top_p=0.9 \ + custom_reward_function.path=${PWD}/verl_reward_functions.py \ + custom_reward_function.name=compute_score \ + algorithm.use_kl_in_reward=False \ + trainer.critic_warmup=0 \ + trainer.val_before_train=False \ + trainer.logger='["console"]' \ + trainer.project_name=grpo-test \ + trainer.experiment_name=llama3.1-8b-grpo-test \ + trainer.n_gpus_per_node=2 \ + trainer.nnodes=1 \ + trainer.save_freq=100 \ + trainer.test_freq=100 \ + trainer.total_training_steps=10 \ + trainer.default_local_dir=/scratch/bfqt/yurenh2/grpo_test_outputs + +echo "GRPO test complete!" +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_grpo_training.sh b/collaborativeagents/slurm/run_grpo_training.sh new file mode 100755 index 0000000..4f9e3f1 --- /dev/null +++ b/collaborativeagents/slurm/run_grpo_training.sh @@ -0,0 +1,69 @@ +#!/bin/bash +#SBATCH --job-name=grpo_refl +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/grpo_reflection_%j.out +#SBATCH --error=logs/grpo_reflection_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/training/outputs + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +# Use the AWQ 70B model for judge (fits on 2 GPUs) +JUDGE_MODEL="hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4" +JUDGE_PORT=8000 + +# Start vLLM server for judge model (on GPUs 2,3) +echo "=== Starting vLLM judge server ===" +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model "$JUDGE_MODEL" \ + --port $JUDGE_PORT \ + --tensor-parallel-size 2 \ + --max-model-len 8192 \ + --dtype auto \ + --trust-remote-code & + +VLLM_PID=$! +echo "vLLM server PID: $VLLM_PID" + +# Wait for server to be ready +echo "Waiting for vLLM server to start..." +for i in {1..60}; do + if curl -s http://localhost:$JUDGE_PORT/health > /dev/null 2>&1; then + echo "vLLM server is ready!" + break + fi + sleep 10 +done + +# Run GRPO training (on GPUs 0,1) +echo "=== Starting GRPO training ===" +CUDA_VISIBLE_DEVICES=0,1 python collaborativeagents/training/train_grpo.py \ + --model-path collaborativeagents/training/outputs/sft_reflection \ + --data-path collaborativeagents/training/training_data/grpo_training_data.json \ + --output-dir collaborativeagents/training/outputs/grpo_reflection \ + --judge-url "http://localhost:$JUDGE_PORT/v1" \ + --judge-model "$JUDGE_MODEL" \ + --max-steps 200 \ + --learning-rate 1e-6 \ + --num-generations 8 + +# Cleanup +echo "=== Cleanup ===" +kill $VLLM_PID 2>/dev/null || true + +echo "=== GRPO Training Complete ===" +echo "Model saved to: collaborativeagents/training/outputs/grpo_reflection/final" diff --git a/collaborativeagents/slurm/run_rag.sh b/collaborativeagents/slurm/run_rag.sh new file mode 100755 index 0000000..4c8bdf9 --- /dev/null +++ b/collaborativeagents/slurm/run_rag.sh @@ -0,0 +1,57 @@ +#!/bin/bash +#SBATCH --job-name=rag +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=2:00:00 +#SBATCH --output=rag_%j.out +#SBATCH --error=rag_%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a; source .env; set +a +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods rag \ + --datasets math-hard \ + --n-profiles 20 \ + --n-sessions 5 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5 \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 20 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/gpt_user_scale_rag_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_rag_bge_v2.sh b/collaborativeagents/slurm/run_rag_bge_v2.sh new file mode 100755 index 0000000..1776bb9 --- /dev/null +++ b/collaborativeagents/slurm/run_rag_bge_v2.sh @@ -0,0 +1,61 @@ +#!/bin/bash +#SBATCH --job-name=rag_bge_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=2:00:00 +#SBATCH --output=rag_bge_v2_%j.out +#SBATCH --error=rag_bge_v2_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# 70B user simulator on GPUs 0,1 +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +# 8B agent on GPUs 2,3 with 40% memory (leaving room for BGE reranker + embedding) +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.40 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods rag_vector_bge \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/rag_vector_bge_v2_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_rag_v2.sh b/collaborativeagents/slurm/run_rag_v2.sh new file mode 100755 index 0000000..29cd6c2 --- /dev/null +++ b/collaborativeagents/slurm/run_rag_v2.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#SBATCH --job-name=rag_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=2:00:00 +#SBATCH --output=rag_v2_%j.out +#SBATCH --error=rag_v2_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.40 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods rag \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/rag_v2_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_rag_vector.sh b/collaborativeagents/slurm/run_rag_vector.sh new file mode 100755 index 0000000..f12d6dc --- /dev/null +++ b/collaborativeagents/slurm/run_rag_vector.sh @@ -0,0 +1,57 @@ +#!/bin/bash +#SBATCH --job-name=rag_vector +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=128G +#SBATCH --time=2:00:00 +#SBATCH --output=rag_vector_%j.out +#SBATCH --error=rag_vector_%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +set -a; source .env; set +a +pip install --quiet openai python-dotenv json-repair + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.5 --max-model-len 8192 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 90); do + curl -s http://localhost:8003/health > /dev/null 2>&1 && break + sleep 2 +done +echo "vLLM ready." + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 20 \ + --n-sessions 5 \ + --max-turns 8 \ + --use-vllm \ + --use-openai-user \ + --openai-user-model gpt-5 \ + --reward-mode llm \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 20 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/gpt_user_scale_rag_vector_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_rag_vector_3x.sh b/collaborativeagents/slurm/run_rag_vector_3x.sh new file mode 100644 index 0000000..7ca2e3e --- /dev/null +++ b/collaborativeagents/slurm/run_rag_vector_3x.sh @@ -0,0 +1,74 @@ +#!/bin/bash +#SBATCH --job-name=rag_3x +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=4:00:00 +#SBATCH --output=rag_3x_%j.out +#SBATCH --error=rag_3x_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# 70B user simulator on GPUs 0,1 +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +# 8B agent on GPUs 2,3 with 40% memory (leaving room for embedding + reranker) +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.40 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +cd collaborativeagents/scripts + +# Run 3 times with different output directories +for run in 1 2 3; do + echo "=========================================" + echo "Starting Run $run of 3" + echo "=========================================" + + python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/rag_vector_run${run}_$(date +%Y%m%d_%H%M%S) + + echo "Run $run complete" + echo "" +done + +echo "All 3 runs complete!" + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_rag_vector_llm_test.sh b/collaborativeagents/slurm/run_rag_vector_llm_test.sh new file mode 100644 index 0000000..01d9c58 --- /dev/null +++ b/collaborativeagents/slurm/run_rag_vector_llm_test.sh @@ -0,0 +1,76 @@ +#!/bin/bash +#SBATCH --job-name=rvec_llm +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=1:00:00 +#SBATCH --output=rvec_llm_%j.out +#SBATCH --error=rvec_llm_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +# Load OpenAI API key +set -a +source .env +set +a + +# Install openai if not present +pip install --quiet openai python-dotenv + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# 70B user simulator on GPUs 0,1 +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +# 8B agent on GPUs 2,3 with 40% memory (leaving room for embedding + reranker) +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.40 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +# Wait for servers +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +echo "vLLM servers ready." + +cd collaborativeagents/scripts + +# Small test: 2 profiles, 5 sessions each +python run_experiments.py \ + --methods rag_vector_llm \ + --datasets math-hard \ + --n-profiles 2 \ + --n-sessions 5 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 2 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/rag_vector_llm_test_$(date +%Y%m%d_%H%M%S) + +echo "Test complete!" + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_rag_vector_v2.sh b/collaborativeagents/slurm/run_rag_vector_v2.sh new file mode 100755 index 0000000..d8151bb --- /dev/null +++ b/collaborativeagents/slurm/run_rag_vector_v2.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#SBATCH --job-name=ragvec_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=2:00:00 +#SBATCH --output=ragvec_v2_%j.out +#SBATCH --error=ragvec_v2_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.40 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods rag_vector \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/rag_vector_v2_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_reflection.sh b/collaborativeagents/slurm/run_reflection.sh new file mode 100755 index 0000000..0f93941 --- /dev/null +++ b/collaborativeagents/slurm/run_reflection.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#SBATCH --job-name=reflection +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/reflection_%j.out +#SBATCH --error=logs/reflection_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods reflection \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/reflection_$(date +%Y%m%d_%H%M%S) diff --git a/collaborativeagents/slurm/run_reflection_grpo.sh b/collaborativeagents/slurm/run_reflection_grpo.sh new file mode 100755 index 0000000..0d93e91 --- /dev/null +++ b/collaborativeagents/slurm/run_reflection_grpo.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#SBATCH --job-name=reflection_grpo +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/reflection_grpo_%j.out +#SBATCH --error=logs/reflection_grpo_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods reflection_grpo \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/reflection_grpo_$(date +%Y%m%d_%H%M%S) diff --git a/collaborativeagents/slurm/run_reflection_v2.sh b/collaborativeagents/slurm/run_reflection_v2.sh new file mode 100755 index 0000000..5e1528c --- /dev/null +++ b/collaborativeagents/slurm/run_reflection_v2.sh @@ -0,0 +1,126 @@ +#!/bin/bash +#SBATCH --job-name=refl_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=4:00:00 +#SBATCH --output=refl_v2_%j.out +#SBATCH --error=refl_v2_%j.err + +# Reflection experiment v2 - with proper_scaffolding enabled (LLM-based retrieval) +# Uses original CollaborativeAgents prompts for fair reproduction +# H200 node, 5 profiles, 15 sessions + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +# Model paths +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +PORT_USER=8004 +PORT_AGENT=8003 + +echo "=== Starting vLLM servers ===" +echo "Method: reflection (with proper_scaffolding)" +echo "User simulator: $MODEL_70B (70B full-precision)" +echo "Agent: $MODEL_8B (8B)" +date + +# Kill any existing vLLM servers +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +# Start 70B user simulator on GPU 0-1 (TP=2) +echo "Starting 70B user simulator on GPU 0-1..." +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B \ + --port $PORT_USER \ + --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 \ + --max-model-len 16384 \ + --download-dir $HF_HOME \ + --dtype bfloat16 \ + --disable-log-requests & +SERVER_USER_PID=$! + +# Start 8B agent on GPU 2-3 (TP=2) +echo "Starting 8B agent on GPU 2-3..." +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B \ + --port $PORT_AGENT \ + --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.90 \ + --max-model-len 16384 \ + --dtype bfloat16 \ + --disable-log-requests & +SERVER_AGENT_PID=$! + +echo "Waiting for vLLM servers to be ready..." + +# Wait for servers +for i in $(seq 1 120); do + READY_USER=$(curl -s http://localhost:$PORT_USER/health > /dev/null 2>&1 && echo 1 || echo 0) + READY_AGENT=$(curl -s http://localhost:$PORT_AGENT/health > /dev/null 2>&1 && echo 1 || echo 0) + + if [ "$READY_USER" = "1" ] && [ "$READY_AGENT" = "1" ]; then + echo "Both servers ready after $((i*3)) seconds" + break + fi + if [ $((i % 20)) -eq 0 ]; then + echo " Still waiting... user=$READY_USER, agent=$READY_AGENT ($((i*3))s)" + fi + sleep 3 +done + +# Verify health +if ! curl -s http://localhost:$PORT_USER/health > /dev/null; then + echo "ERROR: User server not healthy" + kill $SERVER_USER_PID $SERVER_AGENT_PID 2>/dev/null + exit 1 +fi +if ! curl -s http://localhost:$PORT_AGENT/health > /dev/null; then + echo "ERROR: Agent server not healthy" + kill $SERVER_USER_PID $SERVER_AGENT_PID 2>/dev/null + exit 1 +fi +echo "Both vLLM servers healthy!" + +echo "" +echo "=== Running reflection experiment with proper_scaffolding ===" +echo "Settings: 5 profiles, 15 sessions, math-hard dataset" +date + +cd collaborativeagents/scripts + +# Run reflection: 5 profiles, 15 sessions each +python run_experiments.py \ + --methods reflection \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:$PORT_USER/v1 \ + --vllm-agent-url http://localhost:$PORT_AGENT/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/reflection_v2_$(date +%Y%m%d_%H%M%S) + +echo "" +echo "=== Experiment completed ===" +date + +# Cleanup +kill $SERVER_USER_PID $SERVER_AGENT_PID 2>/dev/null || true diff --git a/collaborativeagents/slurm/run_sft_h200.sh b/collaborativeagents/slurm/run_sft_h200.sh new file mode 100644 index 0000000..3be79d2 --- /dev/null +++ b/collaborativeagents/slurm/run_sft_h200.sh @@ -0,0 +1,64 @@ +#!/bin/bash +#SBATCH --job-name=sft_train +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:4 +#SBATCH --mem=200G +#SBATCH --time=12:00:00 +#SBATCH --output=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/sft_train_%j.out +#SBATCH --error=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/sft_train_%j.err + +# SFT Training on H200 + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PWD}/collaborativeagents/scripts:${PYTHONPATH}" +export WANDB_PROJECT="collaborative-agent-reflection-sft" + +echo "=== SFT Training (H200) ===" +date +nvidia-smi --query-gpu=index,name,memory.total --format=csv + +DATA_PATH="collaborativeagents/training/training_data/sft_training_data.json" +MODEL_PATH="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +OUTPUT_DIR="collaborativeagents/training/outputs/sft_reflection" + +echo "Data: $DATA_PATH" +echo "Model: $MODEL_PATH" +echo "Output: $OUTPUT_DIR" + +# Check data exists +if [ ! -f "$DATA_PATH" ]; then + echo "ERROR: Training data not found at $DATA_PATH" + exit 1 +fi + +echo "" +echo "Training data size: $(wc -c < $DATA_PATH) bytes" +python -c "import json; d=json.load(open('$DATA_PATH')); print(f'Training examples: {len(d)}')" + +mkdir -p $OUTPUT_DIR + +echo "" +echo "Starting SFT training..." +python collaborativeagents/training/train_sft.py \ + --model-path $MODEL_PATH \ + --data-path $DATA_PATH \ + --output-dir $OUTPUT_DIR \ + --num-epochs 4 \ + --learning-rate 1e-6 \ + --batch-size 1 \ + --gradient-accumulation 64 + +echo "" +echo "=== SFT Training Complete ===" +echo "Model saved to: $OUTPUT_DIR" +date diff --git a/collaborativeagents/slurm/run_sft_only.sh b/collaborativeagents/slurm/run_sft_only.sh new file mode 100644 index 0000000..a98ae25 --- /dev/null +++ b/collaborativeagents/slurm/run_sft_only.sh @@ -0,0 +1,64 @@ +#!/bin/bash +#SBATCH --job-name=sft_train +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:4 +#SBATCH --mem=200G +#SBATCH --time=24:00:00 +#SBATCH --output=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/sft_train_%j.out +#SBATCH --error=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/sft_train_%j.err + +# SFT Training only (data already exists) + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PWD}/collaborativeagents/scripts:${PYTHONPATH}" +export WANDB_PROJECT="collaborative-agent-reflection-sft" + +echo "=== SFT Training ===" +date +nvidia-smi --query-gpu=index,name,memory.total --format=csv + +DATA_PATH="collaborativeagents/training/training_data/sft_training_data.json" +MODEL_PATH="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +OUTPUT_DIR="collaborativeagents/training/outputs/sft_reflection" + +echo "Data: $DATA_PATH" +echo "Model: $MODEL_PATH" +echo "Output: $OUTPUT_DIR" + +# Check data exists +if [ ! -f "$DATA_PATH" ]; then + echo "ERROR: Training data not found at $DATA_PATH" + exit 1 +fi + +echo "" +echo "Training data size: $(wc -c < $DATA_PATH) bytes" +python -c "import json; d=json.load(open('$DATA_PATH')); print(f'Training examples: {len(d)}')" + +mkdir -p $OUTPUT_DIR + +echo "" +echo "Starting SFT training..." +python collaborativeagents/training/train_sft.py \ + --model-path $MODEL_PATH \ + --data-path $DATA_PATH \ + --output-dir $OUTPUT_DIR \ + --num-epochs 4 \ + --learning-rate 1e-6 \ + --batch-size 1 \ + --gradient-accumulation 64 + +echo "" +echo "=== SFT Training Complete ===" +echo "Model saved to: $OUTPUT_DIR" +date diff --git a/collaborativeagents/slurm/run_sft_training.sh b/collaborativeagents/slurm/run_sft_training.sh new file mode 100755 index 0000000..8cc9f78 --- /dev/null +++ b/collaborativeagents/slurm/run_sft_training.sh @@ -0,0 +1,43 @@ +#!/bin/bash +#SBATCH --job-name=sft_refl +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:4 +#SBATCH --mem=200G +#SBATCH --time=24:00:00 +#SBATCH --output=logs/sft_reflection_%j.out +#SBATCH --error=logs/sft_reflection_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/training/outputs collaborativeagents/training/training_data + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export WANDB_PROJECT="collaborative-agent-reflection-sft" + +# Step 1: Generate training data from completed experiments +echo "=== Step 1: Generating training data ===" +python collaborativeagents/training/generate_training_data.py \ + --results-dir collaborativeagents/results \ + --output-dir collaborativeagents/training/training_data + +# Step 2: Run SFT training using TRL +echo "=== Step 2: Running SFT training ===" +python collaborativeagents/training/train_sft.py \ + --model-path /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct \ + --data-path collaborativeagents/training/training_data/sft_training_data.json \ + --output-dir collaborativeagents/training/outputs/sft_reflection \ + --num-epochs 4 \ + --learning-rate 1e-6 \ + --batch-size 1 \ + --gradient-accumulation 64 + +echo "=== SFT Training Complete ===" +echo "Model saved to: collaborativeagents/training/outputs/sft_reflection" diff --git a/collaborativeagents/slurm/run_vanilla.sh b/collaborativeagents/slurm/run_vanilla.sh new file mode 100755 index 0000000..b29d3a2 --- /dev/null +++ b/collaborativeagents/slurm/run_vanilla.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#SBATCH --job-name=vanilla +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=48:00:00 +#SBATCH --output=logs/vanilla_%j.out +#SBATCH --error=logs/vanilla_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs collaborativeagents/results + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods vanilla \ + --datasets mmlu,aime,math-hard,humaneval \ + --n-profiles 30 \ + --n-sessions 20 \ + --profile-path ../data/complex_profiles_v2/profiles_100.jsonl \ + --output-dir ../results/vanilla_$(date +%Y%m%d_%H%M%S) diff --git a/collaborativeagents/slurm/run_vanilla_v2.sh b/collaborativeagents/slurm/run_vanilla_v2.sh new file mode 100755 index 0000000..5db6064 --- /dev/null +++ b/collaborativeagents/slurm/run_vanilla_v2.sh @@ -0,0 +1,59 @@ +#!/bin/bash +#SBATCH --job-name=vanilla_v2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:4 +#SBATCH --mem=200G +#SBATCH --time=2:00:00 +#SBATCH --output=vanilla_v2_%j.out +#SBATCH --error=vanilla_v2_%j.err + +set -e +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" +export NCCL_P2P_DISABLE=1 + +MODEL_70B="meta-llama/Llama-3.1-70B-Instruct" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" + +pkill -f "vllm.entrypoints" 2>/dev/null || true +sleep 2 + +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B --port 8004 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.95 --max-model-len 16384 \ + --download-dir $HF_HOME --dtype bfloat16 --disable-log-requests & + +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B --port 8003 --tensor-parallel-size 2 \ + --gpu-memory-utilization 0.90 --max-model-len 16384 \ + --dtype bfloat16 --disable-log-requests & + +for i in $(seq 1 120); do + R1=$(curl -s http://localhost:8004/health > /dev/null 2>&1 && echo 1 || echo 0) + R2=$(curl -s http://localhost:8003/health > /dev/null 2>&1 && echo 1 || echo 0) + [ "$R1" = "1" ] && [ "$R2" = "1" ] && break + sleep 3 +done + +cd collaborativeagents/scripts +python run_experiments.py \ + --methods vanilla \ + --datasets math-hard \ + --n-profiles 5 \ + --n-sessions 20 \ + --use-vllm \ + --vllm-user-url http://localhost:8004/v1 \ + --vllm-agent-url http://localhost:8003/v1 \ + --parallel-profiles 5 \ + --profile-path ../data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir ../results/vanilla_v2_$(date +%Y%m%d_%H%M%S) + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/scale_contextual_forget.sbatch b/collaborativeagents/slurm/scale_contextual_forget.sbatch new file mode 100644 index 0000000..4942a4d --- /dev/null +++ b/collaborativeagents/slurm/scale_contextual_forget.sbatch @@ -0,0 +1,69 @@ +#!/bin/bash +#SBATCH --job-name=ctx_forget +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8-interactive +#SBATCH --gres=gpu:4 +#SBATCH --time=01:00:00 +#SBATCH --mem=200G +#SBATCH --cpus-per-task=32 +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source ~/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export TRANSFORMERS_CACHE=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model:$PYTHONPATH + +PROFILE_PATH="collaborativeagents/data/complex_profiles_v2/profiles_200.jsonl" +AGENT_MODEL="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +USER_MODEL="meta-llama/Llama-3.1-70B-Instruct" + +echo "=== Starting vLLM servers ===" +date + +# User simulator on GPUs 0,1 (70B) +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $USER_MODEL \ + --port 8004 --tensor-parallel-size 2 --gpu-memory-utilization 0.90 \ + --max-model-len 16384 --dtype bfloat16 --download-dir $HF_HOME & + +# Agent on GPUs 2,3 (8B) +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $AGENT_MODEL \ + --port 8003 --tensor-parallel-size 2 --gpu-memory-utilization 0.90 \ + --max-model-len 16384 --dtype bfloat16 & + +# Wait for servers +echo "Waiting for vLLM servers..." +for i in {1..200}; do + if curl -s http://localhost:8004/health > /dev/null 2>&1; then + echo "User simulator (8004) ready after $((i*5)) seconds" + break + fi + sleep 5 +done +for i in {1..60}; do + if curl -s http://localhost:8003/health > /dev/null 2>&1; then + echo "Agent (8003) ready after $((i*5)) seconds" + break + fi + sleep 5 +done +echo "Both vLLM servers ready" +sleep 10 + +# Run contextual with reduced memory limits (4000 tokens, 15 turns) +CUDA_VISIBLE_DEVICES=2,3 python collaborativeagents/scripts/run_experiments.py \ + --methods contextual \ + --n-profiles 5 \ + --n-sessions 15 \ + --output-dir results/scale_test_contextual_forget \ + --profile-path $PROFILE_PATH \ + --datasets math-hard \ + --use-vllm --parallel-profiles 30 --no-batch-processing + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/scale_missing.sbatch b/collaborativeagents/slurm/scale_missing.sbatch new file mode 100644 index 0000000..1ab4310 --- /dev/null +++ b/collaborativeagents/slurm/scale_missing.sbatch @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=scale_miss +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8-interactive +#SBATCH --gres=gpu:4 +#SBATCH --time=01:00:00 +#SBATCH --mem=128G +#SBATCH --cpus-per-task=16 +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source ~/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/.cache/huggingface +export TRANSFORMERS_CACHE=/projects/bfqt/users/yurenh2/.cache/huggingface +export PYTHONPATH=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model:$PYTHONPATH + +# Run rag and reflection for profiles 5 only (start_profile=4, end_profile=5 means profile index 4) +python collaborativeagents/scripts/run_experiments.py \ + --methods rag,reflection \ + --num_profiles 5 \ + --start_profile 4 \ + --sessions_per_profile 15 \ + --output_dir results/scale_test_missing \ + --profile_path collaborativeagents/data/complex_profiles_v2/profiles_200.jsonl \ + --datasets math-hard diff --git a/collaborativeagents/slurm/scale_rag_remaining.sbatch b/collaborativeagents/slurm/scale_rag_remaining.sbatch new file mode 100644 index 0000000..b11bba3 --- /dev/null +++ b/collaborativeagents/slurm/scale_rag_remaining.sbatch @@ -0,0 +1,71 @@ +#!/bin/bash +#SBATCH --job-name=scale_rag2 +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8-interactive +#SBATCH --gres=gpu:4 +#SBATCH --time=01:00:00 +#SBATCH --mem=200G +#SBATCH --cpus-per-task=32 +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +source ~/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export TRANSFORMERS_CACHE=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export PYTHONPATH=/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model:$PYTHONPATH + +PROFILE_PATH="collaborativeagents/data/complex_profiles_v2/profiles_200.jsonl" +AGENT_MODEL="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +USER_MODEL="meta-llama/Llama-3.1-70B-Instruct" + +echo "=== Starting vLLM servers ===" +date + +# User simulator on GPUs 0,1 (70B) +CUDA_VISIBLE_DEVICES=0,1 python -m vllm.entrypoints.openai.api_server \ + --model $USER_MODEL \ + --port 8004 --tensor-parallel-size 2 --gpu-memory-utilization 0.90 \ + --max-model-len 16384 --dtype bfloat16 --download-dir $HF_HOME & + +# Agent on GPUs 2,3 (8B) - lower memory for rag_vector (needs embedding/reranker) +CUDA_VISIBLE_DEVICES=2,3 python -m vllm.entrypoints.openai.api_server \ + --model $AGENT_MODEL \ + --port 8003 --tensor-parallel-size 2 --gpu-memory-utilization 0.40 \ + --max-model-len 16384 --dtype bfloat16 & + +# Wait for servers +echo "Waiting for vLLM servers..." +for i in {1..200}; do + if curl -s http://localhost:8004/health > /dev/null 2>&1; then + echo "User simulator (8004) ready after $((i*5)) seconds" + break + fi + sleep 5 +done +for i in {1..60}; do + if curl -s http://localhost:8003/health > /dev/null 2>&1; then + echo "Agent (8003) ready after $((i*5)) seconds" + break + fi + sleep 5 +done +echo "Both vLLM servers ready" +sleep 10 + +# Run rag_vector - only missing profiles 3,4 (0,1,2 already complete) +# Don't restrict CUDA devices - let PersonalizedLLM handle GPU assignment +python collaborativeagents/scripts/run_experiments.py \ + --methods rag_vector \ + --n-profiles 5 \ + --n-sessions 15 \ + --start-profile 3 \ + --output-dir results/scale_test_remaining \ + --profile-path $PROFILE_PATH \ + --datasets math-hard \ + --use-vllm --parallel-profiles 30 --no-batch-processing + +pkill -f "vllm.entrypoints" 2>/dev/null || true diff --git a/collaborativeagents/slurm/start_model_servers.sh b/collaborativeagents/slurm/start_model_servers.sh new file mode 100644 index 0000000..1f4e177 --- /dev/null +++ b/collaborativeagents/slurm/start_model_servers.sh @@ -0,0 +1,82 @@ +#!/bin/bash +#SBATCH --job-name=model_servers +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuH200x8 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gres=gpu:h200:8 +#SBATCH --mem=400G +#SBATCH --time=24:00:00 +#SBATCH --output=logs/model_servers_%j.out +#SBATCH --error=logs/model_servers_%j.err + +# Start vLLM/sglang model servers for experiments +# - Port 8004: Llama-3.3-70B-Instruct (user simulator + judge) - 4 GPUs +# - Port 8003: Llama-3.1-8B-Instruct (agent) - 2 GPUs + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs directory +mkdir -p collaborativeagents/slurm/logs + +echo "Starting model servers at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +# Activate environment with sglang +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Check GPU availability +nvidia-smi + +# Start 70B model server (user simulator + judge) - needs 4 GPUs for TP +echo "Starting Llama-3.3-70B-Instruct server on port 8004..." +CUDA_VISIBLE_DEVICES=0,1,2,3 python -m sglang.launch_server \ + --model-path meta-llama/Llama-3.3-70B-Instruct \ + --port 8004 \ + --tp-size 4 \ + --context-length 16384 \ + --mem-fraction-static 0.85 \ + 2>&1 | tee logs/server_70b_$SLURM_JOB_ID.log & +SERVER_70B_PID=$! + +# Wait for 70B server to start (takes a few minutes) +echo "Waiting for 70B server to initialize..." +sleep 120 + +# Start 8B model server (agent) - needs 2 GPUs +echo "Starting Llama-3.1-8B-Instruct server on port 8003..." +CUDA_VISIBLE_DEVICES=4,5 python -m sglang.launch_server \ + --model-path models/llama-3.1-8b-instruct \ + --served-model-name meta-llama/Llama-3.1-8B-Instruct \ + --port 8003 \ + --tp-size 2 \ + --context-length 16384 \ + --mem-fraction-static 0.85 \ + 2>&1 | tee logs/server_8b_$SLURM_JOB_ID.log & +SERVER_8B_PID=$! + +echo "Servers starting..." +echo "70B server PID: $SERVER_70B_PID" +echo "8B server PID: $SERVER_8B_PID" + +# Save server info for experiment runner +cat > collaborativeagents/slurm/server_info.txt << EOF +NODE=$SLURMD_NODENAME +JOB_ID=$SLURM_JOB_ID +SERVER_70B_PID=$SERVER_70B_PID +SERVER_8B_PID=$SERVER_8B_PID +USER_API_BASE=http://$SLURMD_NODENAME:8004/v1 +AGENT_API_BASE=http://$SLURMD_NODENAME:8003/v1 +JUDGE_API_BASE=http://$SLURMD_NODENAME:8004/v1 +EOF + +echo "Server info saved to collaborativeagents/slurm/server_info.txt" + +# Wait for both servers +wait $SERVER_70B_PID $SERVER_8B_PID diff --git a/collaborativeagents/slurm/test_70b_pilot.sh b/collaborativeagents/slurm/test_70b_pilot.sh new file mode 100755 index 0000000..3514e86 --- /dev/null +++ b/collaborativeagents/slurm/test_70b_pilot.sh @@ -0,0 +1,48 @@ +#!/bin/bash +#SBATCH --job-name=test_70b +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gres=gpu:nvidia_a100:4 +#SBATCH --mem=200G +#SBATCH --time=01:00:00 +#SBATCH --output=logs/test_70b_%j.out +#SBATCH --error=logs/test_70b_%j.err + +# Pilot test for 70B AWQ user model +# Tests that the model loads without OOM and multi-turn works + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +# Create logs directory +mkdir -p collaborativeagents/slurm/logs + +echo "Starting 70B pilot test at $(date)" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURMD_NODENAME" +echo "GPUs: $CUDA_VISIBLE_DEVICES" + +# Activate environment +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Check GPU availability +nvidia-smi + +# Set HF cache to project space +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +mkdir -p $HF_HOME + +# Add project to path +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +# Run pilot test +cd collaborativeagents/scripts +echo "Running 70B pilot test..." +python test_70b_pilot.py + +echo "Pilot test completed at $(date)" diff --git a/collaborativeagents/slurm/test_extractor.sh b/collaborativeagents/slurm/test_extractor.sh new file mode 100755 index 0000000..252af2c --- /dev/null +++ b/collaborativeagents/slurm/test_extractor.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --job-name=test_ext +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=4 +#SBATCH --gres=gpu:1 +#SBATCH --mem=32G +#SBATCH --time=00:10:00 +#SBATCH --output=logs/test_extractor_%j.out +#SBATCH --error=logs/test_extractor_%j.err + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model +mkdir -p collaborativeagents/slurm/logs + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface + +python collaborativeagents/scripts/test_extractor.py diff --git a/collaborativeagents/slurm/test_multiturn.sh b/collaborativeagents/slurm/test_multiturn.sh new file mode 100755 index 0000000..1bf528d --- /dev/null +++ b/collaborativeagents/slurm/test_multiturn.sh @@ -0,0 +1,38 @@ +#!/bin/bash +#SBATCH --job-name=test_multiturn +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gres=gpu:nvidia_a100:2 +#SBATCH --mem=100G +#SBATCH --time=00:30:00 +#SBATCH --output=logs/test_multiturn_%j.out +#SBATCH --error=logs/test_multiturn_%j.err + +# Quick validation test for multi-turn conversation + +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model + +mkdir -p collaborativeagents/slurm/logs + +echo "Starting test at $(date)" +echo "Job ID: $SLURM_JOB_ID" + +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +nvidia-smi + +export HF_HOME=/projects/bfqt/users/yurenh2/.cache/huggingface +export PYTHONPATH="${PWD}/src:${PWD}/collaborativeagents:${PYTHONPATH}" + +cd collaborativeagents/scripts + +echo "Running multi-turn validation test..." +python test_multiturn.py + +echo "Test completed at $(date)" diff --git a/collaborativeagents/slurm/test_vllm_70b_8b.sh b/collaborativeagents/slurm/test_vllm_70b_8b.sh new file mode 100644 index 0000000..815f267 --- /dev/null +++ b/collaborativeagents/slurm/test_vllm_70b_8b.sh @@ -0,0 +1,167 @@ +#!/bin/bash +#SBATCH --job-name=vllm_bench +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=2 +#SBATCH --time=02:00:00 +#SBATCH --mem=128G +#SBATCH --output=slurm/logs/vllm_bench_70b_8b_%j.out +#SBATCH --error=slurm/logs/vllm_bench_70b_8b_%j.err + +# Realistic benchmark: 70B AWQ user simulator + 8B agent +# Tests actual conversation throughput with both models +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface + +echo "=== Job Info ===" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURM_NODELIST" +date + +echo "" +echo "=== GPU Info ===" +nvidia-smi --query-gpu=index,name,memory.total,memory.free --format=csv + +# Download AWQ 70B model if not complete +echo "" +echo "=== Ensuring AWQ 70B Model is Downloaded ===" +python -c " +from huggingface_hub import snapshot_download +import os +os.environ['HF_HOME'] = '/projects/bfqt/users/yurenh2/hf_cache/huggingface' +print('Checking/downloading hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4...') +path = snapshot_download('hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4') +print(f'Model ready at: {path}') +" + +MODEL_70B_AWQ="hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4" +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +PORT_70B=8004 +PORT_8B=8003 + +echo "" +echo "============================================" +echo "Starting 70B AWQ vLLM Server (GPU 0)" +echo "============================================" +CUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_70B_AWQ \ + --port $PORT_70B \ + --gpu-memory-utilization 0.90 \ + --max-model-len 4096 \ + --disable-log-requests \ + --quantization awq \ + --dtype float16 & +SERVER_70B_PID=$! +echo "70B Server PID: $SERVER_70B_PID" + +echo "" +echo "============================================" +echo "Starting 8B vLLM Server (GPU 1)" +echo "============================================" +CUDA_VISIBLE_DEVICES=1 python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B \ + --port $PORT_8B \ + --gpu-memory-utilization 0.90 \ + --max-model-len 4096 \ + --disable-log-requests \ + --dtype bfloat16 & +SERVER_8B_PID=$! +echo "8B Server PID: $SERVER_8B_PID" + +echo "" +echo "Waiting for servers to start..." + +# Wait for 70B (may take 3-5 minutes) +for i in $(seq 1 120); do + if curl -s http://localhost:$PORT_70B/health > /dev/null 2>&1; then + echo "70B Server ready after $((i*3)) seconds" + break + fi + if [ $((i % 20)) -eq 0 ]; then + echo " Waiting for 70B... ($((i*3)) seconds)" + fi + sleep 3 +done + +# Wait for 8B +for i in $(seq 1 60); do + if curl -s http://localhost:$PORT_8B/health > /dev/null 2>&1; then + echo "8B Server ready after $((i*2)) seconds" + break + fi + sleep 2 +done + +# Check both servers +echo "" +if ! curl -s http://localhost:$PORT_70B/health > /dev/null 2>&1; then + echo "ERROR: 70B server failed to start" + kill $SERVER_70B_PID $SERVER_8B_PID 2>/dev/null + exit 1 +fi +echo "✓ 70B server healthy" + +if ! curl -s http://localhost:$PORT_8B/health > /dev/null 2>&1; then + echo "ERROR: 8B server failed to start" + kill $SERVER_70B_PID $SERVER_8B_PID 2>/dev/null + exit 1 +fi +echo "✓ 8B server healthy" + +echo "" +echo "=== vLLM Server Info ===" +echo "70B model:" +curl -s http://localhost:$PORT_70B/v1/models | python -m json.tool 2>/dev/null | head -10 +echo "" +echo "8B model:" +curl -s http://localhost:$PORT_8B/v1/models | python -m json.tool 2>/dev/null | head -10 + +echo "" +echo "============================================" +echo "Test 1: Individual Model Throughput" +echo "============================================" + +echo "" +echo "--- 70B AWQ Sequential (10 requests) ---" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT_70B/v1 -n 10 + +echo "" +echo "--- 8B Sequential (20 requests) ---" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT_8B/v1 -n 20 + +echo "" +echo "============================================" +echo "Test 2: Full Conversation Benchmark" +echo "============================================" +echo "Running 10 conversations with 70B user simulator + 8B agent..." +python scripts/benchmark_inference.py \ + --mode conversation \ + --url-70b http://localhost:$PORT_70B/v1 \ + --url-8b http://localhost:$PORT_8B/v1 \ + -n 10 + +# Cleanup +echo "" +echo "Cleaning up..." +kill $SERVER_70B_PID $SERVER_8B_PID 2>/dev/null +wait $SERVER_70B_PID $SERVER_8B_PID 2>/dev/null + +echo "" +echo "============================================" +echo "BENCHMARK COMPLETE!" +echo "============================================" +echo "" +echo "Key metrics to compare with paper:" +echo " - Paper: 2000 conversations/hour on H100x8" +echo " - Expected A100x2 with 70B AWQ + 8B: ~100-300 conv/hr" +echo " - Our old code: ~20 conv/hr" +echo "" +echo "If throughput is good, update experiment code to use vLLM." +echo "" +date diff --git a/collaborativeagents/slurm/test_vllm_benchmark.sh b/collaborativeagents/slurm/test_vllm_benchmark.sh new file mode 100644 index 0000000..d812b43 --- /dev/null +++ b/collaborativeagents/slurm/test_vllm_benchmark.sh @@ -0,0 +1,102 @@ +#!/bin/bash +#SBATCH --job-name=vllm_bench +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=1 +#SBATCH --time=00:30:00 +#SBATCH --output=slurm/logs/vllm_bench_%j.out +#SBATCH --error=slurm/logs/vllm_bench_%j.err + +# Benchmark vLLM vs transformers inference speed +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface +export TRANSFORMERS_CACHE=/projects/bfqt/users/yurenh2/hf_cache/huggingface + +echo "=== Job Info ===" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURM_NODELIST" +echo "GPUs: $SLURM_GPUS_ON_NODE" +date + +echo "" +echo "=== GPU Info ===" +nvidia-smi --query-gpu=index,name,memory.total,memory.free --format=csv + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +PORT=8003 + +# ============================================ +# Test 1: Transformers baseline +# ============================================ +echo "" +echo "============================================" +echo "Test 1: Transformers Baseline (10 requests)" +echo "============================================" +python scripts/benchmark_inference.py --mode transformers --model $MODEL_8B -n 10 + +# ============================================ +# Test 2: vLLM server +# ============================================ +echo "" +echo "============================================" +echo "Test 2: Starting vLLM Server" +echo "============================================" + +# Start vLLM server +python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B \ + --port $PORT \ + --gpu-memory-utilization 0.9 \ + --max-model-len 4096 \ + --disable-log-requests & +SERVER_PID=$! +echo "vLLM Server PID: $SERVER_PID" + +# Wait for server to be ready +echo "Waiting for server to start..." +for i in {1..60}; do + if curl -s http://localhost:$PORT/health > /dev/null 2>&1; then + echo "Server ready after $((i*5)) seconds" + break + fi + sleep 5 +done + +# Check if server is up +if ! curl -s http://localhost:$PORT/health > /dev/null 2>&1; then + echo "ERROR: vLLM server failed to start" + kill $SERVER_PID 2>/dev/null || true + exit 1 +fi + +echo "" +echo "============================================" +echo "Test 2a: vLLM Sequential (20 requests)" +echo "============================================" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT/v1 -n 20 + +echo "" +echo "============================================" +echo "Test 2b: vLLM Concurrent (50 requests)" +echo "============================================" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT/v1 -n 50 --concurrent + +# Cleanup +echo "" +echo "Cleaning up..." +kill $SERVER_PID 2>/dev/null || true +wait $SERVER_PID 2>/dev/null || true + +echo "" +echo "============================================" +echo "Benchmark Complete!" +echo "============================================" +echo "" +echo "Target: 2000 conversations/hour (paper on H100x8)" +echo "Our A100x4 should achieve ~500-1000 conv/hr with vLLM" +echo "" +date diff --git a/collaborativeagents/slurm/test_vllm_only.sh b/collaborativeagents/slurm/test_vllm_only.sh new file mode 100644 index 0000000..302952c --- /dev/null +++ b/collaborativeagents/slurm/test_vllm_only.sh @@ -0,0 +1,117 @@ +#!/bin/bash +#SBATCH --job-name=vllm_only +#SBATCH --account=bfqt-delta-gpu +#SBATCH --partition=gpuA100x4 +#SBATCH --nodes=1 +#SBATCH --gpus-per-node=1 +#SBATCH --time=00:45:00 +#SBATCH --mem=64G +#SBATCH --output=slurm/logs/vllm_only_%j.out +#SBATCH --error=slurm/logs/vllm_only_%j.err + +# Test vLLM inference speed ONLY (skip transformers which OOMs) +set -e + +cd /projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents + +# Activate conda environment +source /u/yurenh2/miniforge3/etc/profile.d/conda.sh +conda activate eval + +# Install vLLM if not already installed (requires GPU node for CUDA compilation) +if ! python -c "import vllm" 2>/dev/null; then + echo "Installing vLLM..." + pip install vllm --quiet +fi + +export HF_HOME=/projects/bfqt/users/yurenh2/hf_cache/huggingface + +echo "=== Job Info ===" +echo "Job ID: $SLURM_JOB_ID" +echo "Node: $SLURM_NODELIST" +date + +echo "" +echo "=== GPU Info ===" +nvidia-smi --query-gpu=index,name,memory.total,memory.free --format=csv + +MODEL_8B="/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/models/llama-3.1-8b-instruct" +PORT=8003 + +echo "" +echo "============================================" +echo "Starting vLLM Server for 8B Model" +echo "============================================" + +# Start vLLM server with memory optimization +python -m vllm.entrypoints.openai.api_server \ + --model $MODEL_8B \ + --port $PORT \ + --gpu-memory-utilization 0.85 \ + --max-model-len 4096 \ + --disable-log-requests \ + --dtype bfloat16 & +SERVER_PID=$! +echo "vLLM Server PID: $SERVER_PID" + +# Wait for server to be ready +echo "Waiting for server to start..." +for i in {1..90}; do + if curl -s http://localhost:$PORT/health > /dev/null 2>&1; then + echo "Server ready after $((i*2)) seconds" + break + fi + sleep 2 +done + +# Check if server is up +if ! curl -s http://localhost:$PORT/health > /dev/null 2>&1; then + echo "ERROR: vLLM server failed to start" + cat slurm/logs/vllm_only_${SLURM_JOB_ID}.err | tail -50 + kill $SERVER_PID 2>/dev/null || true + exit 1 +fi + +# Get model info +echo "" +echo "=== vLLM Server Info ===" +curl -s http://localhost:$PORT/v1/models | python -m json.tool 2>/dev/null || echo "Could not get model info" + +echo "" +echo "============================================" +echo "Test 1: vLLM Sequential (20 requests)" +echo "============================================" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT/v1 -n 20 + +echo "" +echo "============================================" +echo "Test 2: vLLM Sequential (50 requests)" +echo "============================================" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT/v1 -n 50 + +echo "" +echo "============================================" +echo "Test 3: vLLM Concurrent 4 workers (50 req)" +echo "============================================" +python scripts/benchmark_inference.py --mode vllm --url http://localhost:$PORT/v1 -n 50 --concurrent + +# Cleanup +echo "" +echo "Cleaning up..." +kill $SERVER_PID 2>/dev/null || true +wait $SERVER_PID 2>/dev/null || true + +echo "" +echo "============================================" +echo "BENCHMARK COMPLETE!" +echo "============================================" +echo "" +echo "Key metrics to compare with paper:" +echo " - Paper: 2000 conversations/hour on H100x8" +echo " - Expected A100x1: ~200-500 conv/hr" +echo " - Our old code: ~20 conv/hr (100x slower)" +echo "" +echo "If vLLM shows good throughput, we need to update" +echo "our experiment code to use vLLM instead of transformers." +echo "" +date |