diff options
Diffstat (limited to 'collaborativeagents/slurm/logs')
39 files changed, 25946 insertions, 0 deletions
diff --git a/collaborativeagents/slurm/logs/all_memory_14360420.err b/collaborativeagents/slurm/logs/all_memory_14360420.err new file mode 100644 index 0000000..93582b3 --- /dev/null +++ b/collaborativeagents/slurm/logs/all_memory_14360420.err @@ -0,0 +1,127 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:56:00,092 - INFO - Loaded dataset: mmlu +2025-12-27 01:56:00,092 - INFO - Loaded dataset: aime +2025-12-27 01:56:00,092 - INFO - Loaded dataset: math-hard +2025-12-27 01:56:00,092 - INFO - Loaded dataset: humaneval +2025-12-27 01:56:00,108 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:56:00,108 - INFO - Running method: all_memory +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.35s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 6.00s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.61s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:14, 3.74s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.53s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.06s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.22s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.39s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.87s/it] +2025-12-27 01:57:24,440 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:05<00:41, 5.24s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:11<00:41, 5.96s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:22<00:49, 8.20s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:33<00:45, 9.10s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:51<00:49, 12.45s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:01<00:35, 11.68s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:08<00:20, 10.17s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:15<00:08, 9.00s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:17<00:00, 6.98s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:17<00:00, 8.64s/it] +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-27 02:02:45,722 - WARNING - User agent failed to respond at turn 4 +2025-12-27 02:09:29,423 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:38:42,128 - WARNING - User agent failed to respond at turn 2 +2025-12-27 02:51:47,964 - INFO - Profile 2/30 +2025-12-27 03:33:09,939 - INFO - Profile 3/30 +2025-12-27 03:37:34,526 - WARNING - User agent failed to respond at turn 4 +2025-12-27 04:17:59,068 - WARNING - User agent failed to respond at turn 6 +2025-12-27 04:27:33,697 - INFO - Profile 4/30 +2025-12-27 04:54:07,594 - WARNING - User agent failed to respond at turn 4 +2025-12-27 05:15:55,995 - WARNING - User agent failed to respond at turn 3 +2025-12-27 05:31:32,714 - INFO - Profile 5/30 +2025-12-27 06:00:08,138 - INFO - Profile 6/30 +2025-12-27 06:26:50,737 - WARNING - User agent failed to respond at turn 4 +2025-12-27 06:48:12,610 - INFO - Profile 7/30 +2025-12-27 06:50:45,060 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:09:39,293 - WARNING - User agent failed to respond at turn 6 +2025-12-27 07:21:31,085 - WARNING - User agent failed to respond at turn 4 +2025-12-27 07:31:39,376 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:50:16,030 - INFO - Profile 8/30 +2025-12-27 08:08:45,945 - WARNING - User agent failed to respond at turn 5 +2025-12-27 08:12:05,648 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:35:55,463 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:44:51,942 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:54:22,336 - INFO - Profile 9/30 +2025-12-27 09:27:31,666 - WARNING - User agent failed to respond at turn 7 +2025-12-27 09:38:06,322 - INFO - Profile 10/30 +2025-12-27 10:42:39,399 - INFO - Profile 11/30 +2025-12-27 11:30:24,395 - WARNING - User agent failed to respond at turn 6 +2025-12-27 11:47:24,803 - WARNING - User agent failed to respond at turn 4 +2025-12-27 11:47:24,805 - INFO - Profile 12/30 +2025-12-27 11:54:27,915 - WARNING - User agent failed to respond at turn 8 +2025-12-27 12:45:25,244 - INFO - Profile 13/30 +2025-12-27 13:25:06,083 - INFO - Profile 14/30 +2025-12-27 13:59:04,923 - WARNING - User agent failed to respond at turn 4 +2025-12-27 14:06:28,017 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:08:43,353 - INFO - Profile 15/30 +2025-12-27 14:19:46,215 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:56:28,404 - WARNING - User agent failed to respond at turn 5 +2025-12-27 15:02:31,026 - INFO - Profile 16/30 +2025-12-27 15:41:30,611 - INFO - Profile 17/30 +2025-12-27 16:22:32,360 - WARNING - User agent failed to respond at turn 3 +2025-12-27 16:25:57,033 - WARNING - User agent failed to respond at turn 3 +2025-12-27 16:28:31,989 - WARNING - User agent failed to respond at turn 0 +2025-12-27 16:35:53,597 - WARNING - User agent failed to respond at turn 3 +2025-12-27 16:35:53,598 - INFO - Profile 18/30 +2025-12-27 16:53:53,194 - WARNING - User agent failed to respond at turn 2 +2025-12-27 16:59:54,367 - WARNING - User agent failed to respond at turn 2 +2025-12-27 17:26:40,246 - INFO - Profile 19/30 +2025-12-27 17:29:49,305 - WARNING - User agent failed to respond at turn 3 +2025-12-27 17:34:00,096 - WARNING - User agent failed to respond at turn 6 +2025-12-27 18:29:51,532 - WARNING - User agent failed to respond at turn 3 +2025-12-27 18:34:52,391 - INFO - Profile 20/30 +2025-12-27 19:24:35,799 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:40:55,850 - INFO - Profile 21/30 +2025-12-27 19:45:01,108 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:56:02,480 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:59:59,405 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:12:20,145 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:16:09,511 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:24:18,687 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:41:10,014 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:45:51,169 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:54:31,036 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:59:18,893 - WARNING - User agent failed to respond at turn 3 +2025-12-27 21:09:55,958 - WARNING - User agent failed to respond at turn 3 +2025-12-27 21:09:55,960 - INFO - Profile 22/30 +2025-12-27 21:39:02,331 - WARNING - User agent failed to respond at turn 6 +2025-12-27 22:07:27,301 - INFO - Profile 23/30 +2025-12-27 23:18:34,663 - INFO - Profile 24/30 +2025-12-27 23:36:19,489 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:40:12,672 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:53:47,032 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:10:26,896 - INFO - Profile 25/30 +2025-12-28 00:17:50,523 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:34:10,018 - WARNING - User agent failed to respond at turn 5 +2025-12-28 00:56:33,834 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:10:01,462 - INFO - Profile 26/30 +2025-12-28 01:26:26,996 - WARNING - User agent failed to respond at turn 5 +2025-12-28 01:32:18,640 - WARNING - User agent failed to respond at turn 5 +2025-12-28 01:43:28,926 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:54:35,490 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:58:33,822 - WARNING - User agent failed to respond at turn 5 +2025-12-28 02:15:30,859 - INFO - Profile 27/30 +2025-12-28 02:29:36,631 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:33:19,067 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:46:48,847 - WARNING - User agent failed to respond at turn 6 +2025-12-28 02:55:44,372 - WARNING - User agent failed to respond at turn 3 +2025-12-28 03:25:07,911 - INFO - Profile 28/30 +2025-12-28 04:17:59,003 - WARNING - User agent failed to respond at turn 7 +2025-12-28 04:28:02,080 - INFO - Profile 29/30 +2025-12-28 04:33:42,783 - WARNING - User agent failed to respond at turn 2 +2025-12-28 04:39:07,085 - WARNING - User agent failed to respond at turn 4 +2025-12-28 04:42:42,813 - WARNING - User agent failed to respond at turn 4 +2025-12-28 05:10:55,776 - WARNING - User agent failed to respond at turn 3 +2025-12-28 05:30:48,797 - INFO - Profile 30/30 +2025-12-28 06:06:54,294 - WARNING - User agent failed to respond at turn 3 +2025-12-28 06:19:26,579 - INFO - Report saved to ../results/all_memory_20251227_015537/20251227_015600/report.md diff --git a/collaborativeagents/slurm/logs/contextual_14360423.err b/collaborativeagents/slurm/logs/contextual_14360423.err new file mode 100644 index 0000000..ae2ad9b --- /dev/null +++ b/collaborativeagents/slurm/logs/contextual_14360423.err @@ -0,0 +1,139 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 02:02:06,308 - INFO - Loaded dataset: mmlu +2025-12-27 02:02:06,308 - INFO - Loaded dataset: aime +2025-12-27 02:02:06,308 - INFO - Loaded dataset: math-hard +2025-12-27 02:02:06,308 - INFO - Loaded dataset: humaneval +2025-12-27 02:02:06,329 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 02:02:06,329 - INFO - Running method: contextual +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:21, 7.14s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:12, 6.50s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.40s/it] +2025-12-27 02:02:32,180 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:04<00:33, 4.15s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:08<00:31, 4.52s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:13<00:28, 4.68s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:18<00:24, 4.81s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:23<00:19, 4.88s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:28<00:14, 4.91s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [00:33<00:09, 4.97s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [00:37<00:04, 4.44s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 3.63s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 4.34s/it] +2025-12-27 02:19:52,764 - WARNING - User agent failed to respond at turn 8 +2025-12-27 02:34:35,957 - WARNING - User agent failed to respond at turn 9 +2025-12-27 02:42:19,862 - WARNING - User agent failed to respond at turn 6 +2025-12-27 02:50:50,883 - WARNING - User agent failed to respond at turn 4 +2025-12-27 03:04:43,668 - WARNING - User agent failed to respond at turn 8 +2025-12-27 03:18:17,893 - WARNING - User agent failed to respond at turn 8 +2025-12-27 03:31:55,124 - WARNING - User agent failed to respond at turn 9 +2025-12-27 03:47:00,955 - WARNING - User agent failed to respond at turn 9 +2025-12-27 04:14:22,548 - WARNING - User agent failed to respond at turn 9 +2025-12-27 04:37:23,679 - WARNING - User agent failed to respond at turn 8 +2025-12-27 04:40:56,780 - WARNING - User agent failed to respond at turn 2 +2025-12-27 05:05:21,340 - WARNING - User agent failed to respond at turn 8 +2025-12-27 05:19:26,915 - WARNING - User agent failed to respond at turn 8 +2025-12-27 05:34:10,737 - WARNING - User agent failed to respond at turn 10 +2025-12-27 05:46:47,804 - WARNING - User agent failed to respond at turn 5 +'(ReadTimeoutError("HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)"), '(Request ID: 65a54fd4-dc47-449f-99f1-8b72cd64f258)')' thrown while requesting HEAD https://huggingface.co/datasets/lighteval/MATH-Hard/resolve/cf0716b8bafa192bcb6b455b0679538787dc43f0/.huggingface.yaml +2025-12-27 05:47:08,075 - WARNING - '(ReadTimeoutError("HTTPSConnectionPool(host='huggingface.co', port=443): Read timed out. (read timeout=10)"), '(Request ID: 65a54fd4-dc47-449f-99f1-8b72cd64f258)')' thrown while requesting HEAD https://huggingface.co/datasets/lighteval/MATH-Hard/resolve/cf0716b8bafa192bcb6b455b0679538787dc43f0/.huggingface.yaml +Retrying in 1s [Retry 1/5]. +2025-12-27 05:47:08,075 - WARNING - Retrying in 1s [Retry 1/5]. +2025-12-27 05:47:31,159 - INFO - Profile 2/30 +2025-12-27 06:06:02,584 - WARNING - User agent failed to respond at turn 8 +2025-12-27 06:28:56,672 - WARNING - User agent failed to respond at turn 9 +2025-12-27 06:33:43,493 - WARNING - User agent failed to respond at turn 3 +2025-12-27 06:55:51,361 - WARNING - User agent failed to respond at turn 8 +2025-12-27 07:04:21,768 - WARNING - User agent failed to respond at turn 9 +2025-12-27 07:36:32,573 - WARNING - User agent failed to respond at turn 6 +2025-12-27 08:03:58,605 - WARNING - User agent failed to respond at turn 10 +2025-12-27 08:31:36,913 - WARNING - User agent failed to respond at turn 8 +2025-12-27 08:45:06,689 - WARNING - User agent failed to respond at turn 10 +2025-12-27 08:57:40,880 - WARNING - User agent failed to respond at turn 8 +2025-12-27 09:09:30,300 - WARNING - User agent failed to respond at turn 10 +2025-12-27 09:09:30,302 - INFO - Profile 3/30 +2025-12-27 09:23:09,110 - WARNING - User agent failed to respond at turn 9 +2025-12-27 09:38:33,087 - WARNING - User agent failed to respond at turn 10 +2025-12-27 09:55:31,104 - WARNING - User agent failed to respond at turn 10 +2025-12-27 10:08:01,546 - WARNING - User agent failed to respond at turn 10 +2025-12-27 10:11:50,915 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:20:33,908 - WARNING - User agent failed to respond at turn 8 +2025-12-27 10:36:06,815 - WARNING - User agent failed to respond at turn 13 +2025-12-27 11:04:44,557 - WARNING - User agent failed to respond at turn 9 +2025-12-27 11:19:08,586 - WARNING - User agent failed to respond at turn 10 +2025-12-27 11:32:30,613 - WARNING - User agent failed to respond at turn 8 +2025-12-27 11:45:28,346 - WARNING - User agent failed to respond at turn 8 +2025-12-27 11:59:10,443 - WARNING - User agent failed to respond at turn 8 +2025-12-27 12:02:44,417 - WARNING - User agent failed to respond at turn 2 +2025-12-27 12:24:05,983 - WARNING - User agent failed to respond at turn 9 +2025-12-27 12:36:47,157 - WARNING - User agent failed to respond at turn 8 +2025-12-27 12:56:50,763 - WARNING - User agent failed to respond at turn 8 +2025-12-27 13:14:23,987 - INFO - Profile 4/30 +2025-12-27 13:41:52,436 - WARNING - User agent failed to respond at turn 8 +2025-12-27 14:02:43,583 - WARNING - User agent failed to respond at turn 8 +2025-12-27 14:06:32,824 - WARNING - User agent failed to respond at turn 2 +2025-12-27 14:23:22,832 - WARNING - User agent failed to respond at turn 14 +2025-12-27 14:37:30,153 - WARNING - User agent failed to respond at turn 8 +2025-12-27 14:51:10,701 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:04:43,741 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:19:48,665 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:33:01,051 - WARNING - User agent failed to respond at turn 8 +2025-12-27 15:49:22,254 - WARNING - User agent failed to respond at turn 8 +2025-12-27 16:03:56,686 - WARNING - User agent failed to respond at turn 8 +2025-12-27 16:20:53,945 - WARNING - User agent failed to respond at turn 7 +2025-12-27 16:49:33,698 - WARNING - User agent failed to respond at turn 8 +2025-12-27 17:04:50,896 - WARNING - User agent failed to respond at turn 9 +2025-12-27 17:31:00,796 - WARNING - User agent failed to respond at turn 8 +2025-12-27 17:31:00,799 - INFO - Profile 5/30 +2025-12-27 18:00:49,422 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:15:01,789 - WARNING - User agent failed to respond at turn 10 +2025-12-27 18:27:47,576 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:40:13,645 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:53:15,915 - WARNING - User agent failed to respond at turn 8 +2025-12-27 19:19:08,201 - WARNING - User agent failed to respond at turn 8 +2025-12-27 19:37:35,846 - WARNING - User agent failed to respond at turn 8 +2025-12-27 19:51:06,794 - WARNING - User agent failed to respond at turn 10 +2025-12-27 20:05:06,821 - WARNING - User agent failed to respond at turn 10 +2025-12-27 20:17:39,488 - WARNING - User agent failed to respond at turn 8 +2025-12-27 20:31:13,429 - WARNING - User agent failed to respond at turn 7 +2025-12-27 20:53:38,048 - WARNING - User agent failed to respond at turn 9 +2025-12-27 21:06:30,110 - WARNING - User agent failed to respond at turn 9 +2025-12-27 21:18:58,273 - WARNING - User agent failed to respond at turn 8 +2025-12-27 21:31:52,874 - WARNING - User agent failed to respond at turn 8 +2025-12-27 21:45:02,984 - WARNING - User agent failed to respond at turn 8 +2025-12-27 21:45:02,987 - INFO - Profile 6/30 +2025-12-27 21:59:20,567 - WARNING - User agent failed to respond at turn 12 +2025-12-27 22:12:05,436 - WARNING - User agent failed to respond at turn 9 +2025-12-27 22:24:26,612 - WARNING - User agent failed to respond at turn 8 +2025-12-27 22:37:34,213 - WARNING - User agent failed to respond at turn 8 +2025-12-27 22:41:11,469 - WARNING - User agent failed to respond at turn 2 +2025-12-27 22:54:18,480 - WARNING - User agent failed to respond at turn 8 +2025-12-27 22:57:33,805 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:09:44,313 - WARNING - User agent failed to respond at turn 9 +2025-12-27 23:22:24,512 - WARNING - User agent failed to respond at turn 8 +2025-12-27 23:35:10,690 - WARNING - User agent failed to respond at turn 8 +2025-12-27 23:51:25,271 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:04:32,550 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:17:05,022 - WARNING - User agent failed to respond at turn 7 +2025-12-28 00:29:45,695 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:42:31,155 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:55:35,641 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:09:42,819 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:22:54,235 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:35:57,818 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:35:57,820 - INFO - Profile 7/30 +2025-12-28 02:29:13,033 - WARNING - User agent failed to respond at turn 8 +2025-12-28 02:34:18,064 - WARNING - User agent failed to respond at turn 2 +2025-12-28 02:47:50,033 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:01:20,372 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:17:29,349 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:31:22,192 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:44:57,553 - WARNING - User agent failed to respond at turn 8 +2025-12-28 03:59:52,603 - WARNING - User agent failed to respond at turn 8 +[2025-12-28T04:23:53.546] error: *** JOB 14360423 ON gpua091 CANCELLED AT 2025-12-28T04:23:53 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/full_exp_14357783.err b/collaborativeagents/slurm/logs/full_exp_14357783.err new file mode 100644 index 0000000..a0c7ce5 --- /dev/null +++ b/collaborativeagents/slurm/logs/full_exp_14357783.err @@ -0,0 +1,69 @@ +2025-12-26 02:38:46,860 - INFO - Loaded dataset: gpqa +2025-12-26 02:38:46,861 - INFO - Loaded dataset: aime +2025-12-26 02:38:46,861 - INFO - Loaded dataset: math-hard +2025-12-26 02:38:46,861 - INFO - Loaded dataset: humaneval +2025-12-26 02:38:46,872 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-26 02:38:46,872 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:09<00:27, 9.02s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:11, 5.81s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.42s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.37s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:16, 4.19s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:10, 3.44s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.63s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.44s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.65s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.07s/it] +2025-12-26 02:39:34,147 - INFO - Profile 1/30 +
Generating train split: 0%| | 0/90 [00:00<?, ? examples/s]
Generating train split: 100%|██████████| 90/90 [00:00<00:00, 1116.85 examples/s] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:05<00:41, 5.22s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:13<00:49, 7.11s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:23<00:48, 8.17s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:33<00:44, 8.98s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:45<00:40, 10.19s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:56<00:31, 10.44s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:04<00:19, 9.67s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:08<00:07, 7.94s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:10<00:00, 5.95s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:10<00:00, 7.83s/it] +2025-12-26 02:40:51,255 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,261 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,266 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,271 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,276 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,281 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,286 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,291 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,296 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,301 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,306 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,310 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,315 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,320 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,326 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,330 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,335 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,341 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,345 - WARNING - User agent failed to respond at turn 0 +2025-12-26 02:40:51,350 - WARNING - User agent failed to respond at turn 0 +
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 2304 examples [00:00, 29493.27 examples/s] +
Generating test split: 0 examples [00:00, ? examples/s]
Generating test split: 1324 examples [00:00, 28145.98 examples/s] +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 623, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 608, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 414, in run_all + results = self.run_method(method) + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 367, in run_method + samples = dataset.get_testset() + ^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/datasets_extended.py", line 71, in get_testset + self._test_data = self._load_data("test")[:self.eval_size] + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/datasets_extended.py", line 153, in _load_data + solution=item["answer"], + ~~~~^^^^^^^^^^ +KeyError: 'answer' diff --git a/collaborativeagents/slurm/logs/full_exp_14358390.err b/collaborativeagents/slurm/logs/full_exp_14358390.err new file mode 100644 index 0000000..65669a6 --- /dev/null +++ b/collaborativeagents/slurm/logs/full_exp_14358390.err @@ -0,0 +1,97 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-26 11:03:58,116 - INFO - Loaded dataset: mmlu +2025-12-26 11:03:58,116 - INFO - Loaded dataset: aime +2025-12-26 11:03:58,116 - INFO - Loaded dataset: math-hard +2025-12-26 11:03:58,116 - INFO - Loaded dataset: humaneval +2025-12-26 11:03:58,213 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-26 11:03:58,214 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:16, 5.63s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.11s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 5.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.87s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.49s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:23, 5.84s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.19s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:19<00:04, 4.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:21<00:00, 3.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:21<00:00, 4.25s/it] +2025-12-26 11:04:49,945 - INFO - Profile 1/30 +
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 1179.52 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3170.95 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1567.85 examples/s] +
Generating test split: 0%| | 0/100 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 100/100 [00:00<00:00, 21317.94 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3131.14 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1635.46 examples/s] +
Generating test split: 0%| | 0/102 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 102/102 [00:00<00:00, 22671.91 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 3275.40 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1562.01 examples/s] +
Generating test split: 0%| | 0/126 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 126/126 [00:00<00:00, 22251.89 examples/s] +
Generating validation split: 0%| | 0/14 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 14/14 [00:00<00:00, 3442.39 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1582.64 examples/s] +
Generating test split: 0%| | 0/151 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 151/151 [00:00<00:00, 33935.59 examples/s] +
Generating validation split: 0%| | 0/17 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 17/17 [00:00<00:00, 5068.47 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1145.67 examples/s] +
Generating test split: 0%| | 0/112 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 112/112 [00:00<00:00, 28756.25 examples/s] +
Generating validation split: 0%| | 0/11 [00:00<?, ? examples/s]
Generating validation split: 100%|██████████| 11/11 [00:00<00:00, 1310.68 examples/s] +
Generating dev split: 0%| | 0/5 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 5/5 [00:00<00:00, 1548.86 examples/s] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:05<00:44, 5.50s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [04:02<16:31, 141.68s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [04:07<07:54, 79.14s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [04:13<04:11, 50.20s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [04:18<02:16, 34.13s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [04:24<01:13, 24.38s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [05:50<01:29, 44.64s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [05:53<00:31, 31.48s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [05:55<00:00, 22.11s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [05:55<00:00, 39.48s/it] +2025-12-26 11:22:36,555 - WARNING - User agent failed to respond at turn 3 +2025-12-26 11:38:11,820 - WARNING - User agent failed to respond at turn 4 +2025-12-26 11:49:23,100 - WARNING - User agent failed to respond at turn 2 +2025-12-26 12:02:46,057 - INFO - Profile 2/30 +2025-12-26 12:15:14,260 - WARNING - User agent failed to respond at turn 5 +2025-12-26 12:43:01,887 - INFO - Profile 3/30 +2025-12-26 13:16:25,157 - WARNING - User agent failed to respond at turn 3 +2025-12-26 13:19:41,611 - WARNING - User agent failed to respond at turn 4 +2025-12-26 13:31:32,934 - INFO - Profile 4/30 +2025-12-26 13:44:18,881 - WARNING - User agent failed to respond at turn 2 +2025-12-26 13:50:24,735 - WARNING - User agent failed to respond at turn 3 +2025-12-26 14:22:35,201 - WARNING - User agent failed to respond at turn 4 +2025-12-26 14:28:33,134 - INFO - Profile 5/30 +2025-12-26 15:08:47,500 - INFO - Profile 6/30 +2025-12-26 15:11:23,025 - WARNING - User agent failed to respond at turn 4 +2025-12-26 15:16:01,352 - WARNING - User agent failed to respond at turn 3 +2025-12-26 15:42:13,161 - WARNING - User agent failed to respond at turn 4 +2025-12-26 15:46:22,086 - WARNING - User agent failed to respond at turn 6 +2025-12-26 15:56:17,527 - WARNING - User agent failed to respond at turn 4 +2025-12-26 16:07:50,432 - WARNING - User agent failed to respond at turn 6 +2025-12-26 16:07:50,434 - INFO - Profile 7/30 +2025-12-26 16:17:55,588 - WARNING - User agent failed to respond at turn 2 +2025-12-26 16:33:05,425 - WARNING - User agent failed to respond at turn 3 +2025-12-26 16:56:24,660 - WARNING - User agent failed to respond at turn 3 +2025-12-26 17:08:04,806 - INFO - Profile 8/30 +2025-12-26 18:12:23,818 - WARNING - User agent failed to respond at turn 3 +2025-12-26 18:15:59,440 - INFO - Profile 9/30 +2025-12-26 18:25:15,857 - WARNING - User agent failed to respond at turn 3 +2025-12-26 18:27:49,419 - WARNING - User agent failed to respond at turn 3 +2025-12-26 18:57:08,009 - INFO - Profile 10/30 +2025-12-26 19:36:53,452 - WARNING - User agent failed to respond at turn 2 +2025-12-26 19:54:08,446 - INFO - Profile 11/30 +2025-12-26 20:09:55,735 - WARNING - User agent failed to respond at turn 3 +2025-12-26 20:40:31,861 - INFO - Profile 12/30 +2025-12-26 21:14:56,537 - WARNING - User agent failed to respond at turn 3 +2025-12-26 21:28:00,763 - WARNING - User agent failed to respond at turn 3 +2025-12-26 21:28:00,764 - INFO - Profile 13/30 +2025-12-26 22:12:17,605 - INFO - Profile 14/30 +2025-12-26 22:35:24,416 - WARNING - User agent failed to respond at turn 4 +2025-12-26 22:59:55,984 - INFO - Profile 15/30 +2025-12-26 23:10:51,206 - WARNING - User agent failed to respond at turn 2 +2025-12-26 23:23:48,320 - WARNING - User agent failed to respond at turn 3 +2025-12-26 23:41:06,303 - INFO - Profile 16/30 +2025-12-27 00:26:43,745 - INFO - Profile 17/30 +2025-12-27 00:51:28,080 - WARNING - User agent failed to respond at turn 5 +2025-12-27 01:04:15,959 - WARNING - User agent failed to respond at turn 2 +2025-12-27 01:11:14,362 - WARNING - User agent failed to respond at turn 3 +2025-12-27 01:16:42,254 - WARNING - User agent failed to respond at turn 3 +2025-12-27 01:22:35,316 - INFO - Profile 18/30 +[2025-12-27T01:24:47.209] error: *** JOB 14358390 ON gpua035 CANCELLED AT 2025-12-27T01:24:47 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/rag_14360421.err b/collaborativeagents/slurm/logs/rag_14360421.err new file mode 100644 index 0000000..de70ef7 --- /dev/null +++ b/collaborativeagents/slurm/logs/rag_14360421.err @@ -0,0 +1,9442 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:58:02,188 - INFO - Loaded dataset: mmlu +2025-12-27 01:58:02,188 - INFO - Loaded dataset: aime +2025-12-27 01:58:02,188 - INFO - Loaded dataset: math-hard +2025-12-27 01:58:02,188 - INFO - Loaded dataset: humaneval +2025-12-27 01:58:02,201 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:58:02,201 - INFO - Running method: rag +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:21, 7.24s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.15s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.70s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.99s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.17s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.07s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.14s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.82s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.24s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.58s/it] +2025-12-27 01:58:53,808 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:04<00:38, 4.87s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:12<00:45, 6.46s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:23<00:50, 8.41s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:33<00:46, 9.26s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:47<00:43, 10.88s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:57<00:31, 10.56s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:04<00:18, 9.48s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:12<00:08, 8.89s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:15<00:00, 7.14s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:15<00:00, 8.40s/it] +[2025-12-27T02:03:19.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:06:19.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:07:51.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:12.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:15.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:25.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:43.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:47.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:08:49.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:11.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:15.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:17.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:48.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:50.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:09:52.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:33.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:35.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:37.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:39.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:43.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:10:45.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:12.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:15.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:25.126] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:43.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:47.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:11:49.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:14.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:16.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:18.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:42.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:44.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:12:46.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:13:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:13.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:17.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:19.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:40.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:44.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:14:46.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:14.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:16.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:26.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:47.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:49.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:15:51.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:16.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:18.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:20.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:41.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:43.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:16:45.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:14.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:16.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:26.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:44.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:48.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:17:50.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:15.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:17.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:19.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:42.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:44.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:18:46.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:14.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:16.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:26.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:46.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:48.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:19:50.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:15.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:17.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:19.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:42.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:20:46.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:14.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:17.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:27.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:47.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:49.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:21:51.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:18.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:20.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:41.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:44.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:22:46.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:12.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:16.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:26.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:46.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:48.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:23:50.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:13.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:17.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:19.624] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:42.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:44.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:24:46.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:14.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:16.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:26.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:44.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:48.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:25:50.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:13.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:17.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:20.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:26:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:27:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:13.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:17.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:28:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:14.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:44.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:29:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:13.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:42.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:44.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:30:46.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:12.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:45.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:49.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:31:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:14.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:18.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:20.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:41.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:32:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:14.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:19.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:26.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:46.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:48.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:33:50.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:13.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:17.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:19.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:42.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:34:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:12.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:16.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:26.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:46.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:48.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:35:50.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:15.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:17.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:42.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:44.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:36:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:14.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:16.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:26.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:46.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:48.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:37:50.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:15.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:17.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:19.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:42.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:44.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:38:46.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:14.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:16.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:26.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:46.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:48.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:39:50.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:15.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:17.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:19.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:42.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:44.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:40:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:14.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:16.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:26.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:44.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:48.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:41:50.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:16.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:18.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:20.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:42:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:46.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:43:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:13.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:17.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:19.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:42.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:44.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:44:46.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:14.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:16.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:26.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:44.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:48.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:45:50.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:15.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:17.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:19.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:41.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:43.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:46:45.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:12.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:15.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:25.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:46.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:48.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:47:50.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:15.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:17.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:19.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:42.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:44.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:48:46.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:12.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:16.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:26.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:48.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:49:50.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:13.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:17.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:19.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:42.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:44.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:50:46.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:12.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:16.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:44.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:51:50.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:15.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:18.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:20.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:52:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:16.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:53:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:15.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:40.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:54:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:14.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:19.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:46.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:55:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:13.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:17.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:56:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:12.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:16.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:45.098] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:57:51.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:18.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:39.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:43.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:58:45.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:12.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:16.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:26.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:44.369] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:48.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T02:59:50.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:15.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:17.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:19.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:42.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:44.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:00:46.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:14.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:26.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:44.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:48.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:01:50.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:13.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:17.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:19.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:42.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:44.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:02:46.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:14.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:16.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:26.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:47.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:49.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:03:51.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:14.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:18.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:20.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:41.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:43.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:04:45.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:14.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:16.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:26.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:46.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:48.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:05:50.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:13.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:17.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:19.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:40.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:44.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:06:46.337] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:12.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:16.615] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:26.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:44.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:48.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:07:50.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:12.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:16.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:18.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:08:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:09:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:15.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:17.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:42.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:44.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:10:46.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:12.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:16.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:26.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:44.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:48.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:11:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:15.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:17.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:19.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:42.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:44.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:12:46.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:14.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:16.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:47.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:49.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:13:51.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:16.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:18.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:20.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:41.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:43.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:14:45.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:12.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:16.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:26.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:46.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:48.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:15:50.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:13.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:17.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:19.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:42.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:44.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:16:46.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:12.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:16.492] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:26.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:45.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:47.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:17:49.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:13.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:15.816] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:17.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:48.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:50.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:18:53.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:20.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:33.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:35.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:42.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:44.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:19:46.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:14.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:16.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:26.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:46.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:49.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:20:51.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:14.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:18.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:20.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:41.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:43.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:21:45.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:14.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:19.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:26.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:46.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:48.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:22:50.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:15.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:17.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:19.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:41.180] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:43.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:23:45.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:12.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:16.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:26.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:46.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:48.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:24:50.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:15.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:17.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:19.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:42.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:44.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:25:46.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:14.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:21.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:26.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:46.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:48.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:26:50.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:13.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:17.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:19.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:42.037] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:44.046] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:27:46.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:14.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:16.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:26.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:44.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:48.532] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:28:50.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:13.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:17.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:19.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:42.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:44.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:29:46.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:12.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:16.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:26.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:46.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:48.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:30:50.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:15.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:18.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:20.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:31:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:14.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:19.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:44.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:32:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:15.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:17.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:40.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:43.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:33:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:16.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:34:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:15.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:19.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:35:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:12.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:16.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:26.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:47.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:49.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:36:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:13.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:17.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:19.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:42.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:44.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:37:46.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:12.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:16.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:26.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:46.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:48.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:38:50.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:17.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:19.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:42.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:44.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:39:46.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:12.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:16.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:46.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:48.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:40:50.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:15.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:17.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:20.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:41:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:45.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:47.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:42:49.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:12.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:16.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:18.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:42.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:44.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:43:46.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:26.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:46.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:44:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:15.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:40.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:45:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:14.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:26.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:45.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:46:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:14.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:18.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:20.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:47:45.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:12.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:16.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:26.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:46.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:48.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:48:50.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:15.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:17.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:19.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:42.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:44.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:49:46.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:14.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:16.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:26.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:46.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:48.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:50:50.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:13.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:17.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:19.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:41.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:43.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:51:45.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:15.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:17.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:27.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:45.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:49.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:52:51.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:16.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:18.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:20.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:40.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:44.047] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:53:46.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:12.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:16.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:26.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:44.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:48.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:54:50.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:13.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:17.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:19.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:42.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:44.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:55:46.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:14.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:16.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:26.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:44.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:48.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:56:50.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:15.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:18.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:20.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:57:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:58:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T03:59:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:14.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:16.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:46.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:00:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:17.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:19.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:42.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:44.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:01:46.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:14.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:16.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:47.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:02:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:14.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:18.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:41.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:43.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:03:45.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:14.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:16.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:26.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:44.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:48.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:04:50.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:13.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:17.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:19.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:42.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:05:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:14.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:16.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:26.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:46.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:48.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:06:50.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:13.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:42.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:44.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:07:46.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:14.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:24.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:26.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:47.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:49.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:08:51.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:16.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:18.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:20.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:41.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:43.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:09:45.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:12.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:16.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:26.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:48.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:10:50.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:13.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:17.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:42.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:44.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:11:46.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:12.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:16.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:26.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:46.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:48.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:12:50.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:12.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:16.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:18.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:13:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:44.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:14:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:40.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:43.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:15:45.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:12.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:16.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:26.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:48.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:16:50.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:15.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:19.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:44.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:17:46.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:15.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:25.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:46.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:48.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:18:50.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:15.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:17.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:19.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:40.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:44.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:19:46.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:14.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:16.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:26.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:46.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:48.514] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:20:50.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:13.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:17.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:19.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:42.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:44.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:21:46.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:12.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:16.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:26.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:44.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:22:50.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:15.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:18.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:20.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:23:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:24:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:25:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:14.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:24.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:44.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:26:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:13.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:17.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:27:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:14.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:21.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:26.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:45.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:28:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:14.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:17.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:19.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:42.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:44.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:29:46.039] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:14.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:16.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:26.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:44.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:48.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:30:50.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:15.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:17.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:19.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:42.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:44.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:31:46.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:14.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:19.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:26.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:44.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:48.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:32:50.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:13.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:18.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:20.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:33:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:34:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:41.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:35:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:14.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:16.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:26.562] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:46.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:48.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:36:50.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:13.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:17.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:19.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:44.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:37:46.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:12.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:16.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:26.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:38:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:16.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:18.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:20.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:41.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:43.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:39:45.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:14.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:19.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:26.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:46.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:48.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:40:50.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:13.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:17.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:19.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:40.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:44.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:41:46.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:12.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:26.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:46.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:48.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:42:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:15.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:19.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:42.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:44.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:43:46.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:14.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:16.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:26.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:47.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:49.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:44:51.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:14.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:18.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:20.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:41.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:43.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:45:45.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:12.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:16.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:26.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:45.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:47.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:46:49.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:12.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:18.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:42.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:44.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:47:46.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:12.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:16.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:26.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:44.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:48.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:48:50.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:16.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:18.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:20.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:49:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:44.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:50:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:51:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:46.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:52:50.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:19.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:53:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:26.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:47.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:54:51.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:16.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:20.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:40.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:42.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:55:44.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:12.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:16.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:26.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:44.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:48.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:56:50.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:13.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:17.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:19.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:42.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:44.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:57:46.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:12.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:16.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:46.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:48.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:58:50.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:15.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:17.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:20.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T04:59:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:00:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:01:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:47.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:02:49.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:11.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:15.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:17.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:49.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:51.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:03:53.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:34.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:36.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:38.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:40.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:44.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:04:46.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:12.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:16.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:26.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:46.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:48.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:05:50.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:15.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:17.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:19.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:42.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:44.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:06:46.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:14.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:19.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:26.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:47.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:49.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:07:51.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:16.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:18.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:20.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:41.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:43.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:08:45.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:12.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:16.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:26.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:48.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:09:50.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:13.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:17.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:19.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:42.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:44.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:10:46.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:14.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:19.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:26.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:44.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:47.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:11:49.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:14.900] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:16.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:18.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:12:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:16.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:48.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:13:50.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:15.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:44.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:14:46.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:12.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:16.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:26.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:45.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:47.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:15:49.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:12.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:16.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:18.785] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:16:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:14.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:16.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:47.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:49.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:17:51.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:14.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:18.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:20.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:18:45.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:12.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:16.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:26.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:46.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:48.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:19:50.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:13.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:17.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:42.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:44.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:20:46.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:14.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:16.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:26.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:46.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:48.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:21:50.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:13.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:17.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:42.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:44.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:22:46.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:12.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:16.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:26.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:47.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:49.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:23:51.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:16.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:18.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:20.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:41.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:43.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:24:45.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:14.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:16.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:26.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:46.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:48.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:25:50.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:13.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:17.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:19.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:42.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:44.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:26:46.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:12.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:16.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:26.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:46.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:48.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:27:50.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:13.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:19.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:42.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:44.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:28:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:14.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:27.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:47.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:49.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:29:51.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:18.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:20.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:39.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:44.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:30:46.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:14.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:16.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:26.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:46.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:48.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:31:50.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:15.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:17.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:19.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:42.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:44.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:32:46.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:14.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:16.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:26.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:46.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:48.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:33:50.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:13.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:17.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:19.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:34:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:46.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:35:50.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:13.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:19.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:36:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:19.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:37:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:15.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:17.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:42.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:44.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:38:46.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:14.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:19.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:26.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:45.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:39:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:16.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:18.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:20.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:41.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:43.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:40:45.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:14.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:16.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:26.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:48.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:41:50.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:13.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:17.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:40.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:44.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:42:46.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:12.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:16.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:26.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:46.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:48.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:43:50.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:13.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:17.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:19.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:42.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:44.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:44:46.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:14.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:16.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:26.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:47.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:49.180] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:45:51.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:16.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:18.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:20.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:39.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:43.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:46:45.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:14.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:26.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:44.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:48.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:47:50.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:15.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:17.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:19.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:42.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:44.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:48:46.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:14.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:16.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:26.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:44.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:48.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:49:50.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:15.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:17.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:19.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:50:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:15.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:20.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:27.031] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:47.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:49.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:51:51.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:16.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:18.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:20.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:42.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:44.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:52:46.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:14.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:16.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:26.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:46.467] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:48.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:53:50.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:17.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:19.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:40.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:44.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:54:46.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:14.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:16.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:44.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:55:50.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:15.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:17.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:20.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:56:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:14.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:21.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:44.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:48.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:57:50.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:15.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:17.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:19.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:42.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:44.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:58:46.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:12.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:16.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:26.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:44.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:48.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T05:59:50.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:13.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:17.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:19.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:40.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:44.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:00:46.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:12.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:16.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:26.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:47.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:49.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:01:51.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:16.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:18.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:20.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:41.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:43.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:02:45.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:12.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:15.098] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:25.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:45.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:47.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:03:49.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:14.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:16.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:18.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:42.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:04:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:12.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:16.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:46.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:05:50.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:15.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:17.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:19.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:40.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:44.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:06:46.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:14.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:16.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:26.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:45.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:49.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:07:51.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:16.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:18.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:20.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:41.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:43.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:08:45.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:14.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:24.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:26.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:45.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:47.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:09:49.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:14.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:16.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:18.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:40.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:44.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:10:46.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:14.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:16.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:26.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:46.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:48.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:11:50.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:16.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:18.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:20.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:12:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:44.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:48.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:13:50.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:17.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:41.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:14:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:12.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:44.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:15:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:44.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:16:46.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:12.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:45.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:17:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:16.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:18.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:41.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:43.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:18:45.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:14.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:16.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:26.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:44.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:48.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:19:50.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:15.505] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:42.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:44.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:20:46.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:14.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:19.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:26.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:44.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:48.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:21:50.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:15.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:17.908] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:19.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:40.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:44.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:22:46.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:14.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:24.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:26.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:47.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:49.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:23:51.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:14.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:18.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:20.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:41.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:43.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:24:45.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:12.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:16.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:26.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:46.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:48.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:25:50.473] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:13.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:17.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:19.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:42.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:44.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:26:46.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:12.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:16.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:26.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:46.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:48.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:27:50.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:15.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:17.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:19.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:40.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:44.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:28:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:15.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:19.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:26.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:46.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:48.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:29:50.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:15.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:17.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:19.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:41.094] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:43.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:30:45.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:14.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:16.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:26.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:44.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:48.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:31:50.708] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:15.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:17.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:19.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:40.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:44.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:32:46.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:12.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:16.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:26.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:45.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:49.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:33:51.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:14.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:18.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:20.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:41.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:43.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:34:45.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:12.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:15.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:25.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:45.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:47.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:35:49.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:12.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:16.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:18.485] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:42.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:44.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:36:46.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:12.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:16.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:26.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:46.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:48.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:37:50.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:15.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:17.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:19.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:42.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:38:46.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:14.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:17.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:27.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:47.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:49.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:39:51.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:16.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:18.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:20.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:41.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:43.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:40:46.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:12.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:16.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:26.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:46.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:48.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:41:50.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:15.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:17.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:19.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:42.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:44.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:42:46.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:14.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:16.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:26.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:44.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:48.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:43:50.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:15.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:17.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:19.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:44:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:15.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:25.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:46.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:45:48.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:11.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:15.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:17.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:46.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:50.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:46:52.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:47:34.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:47:36.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:47:38.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:12.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:19.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:46.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:48:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:40.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:49:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:12.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:16.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:26.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:46.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:48.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:50:50.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:15.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:17.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:19.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:40.045] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:44.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:51:46.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:12.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:16.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:26.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:44.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:48.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:52:50.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:13.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:17.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:19.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:40.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:44.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:53:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:12.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:16.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:26.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:44.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:48.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:54:50.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:16.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:18.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:20.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:55:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:56:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:15.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:40.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:57:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:19.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:48.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:58:50.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:13.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:17.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:19.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:42.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:44.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T06:59:46.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:47.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:00:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:14.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:18.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:39.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:43.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:01:45.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:14.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:21.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:44.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:48.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:02:50.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:15.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:19.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:42.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:44.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:03:46.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:12.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:16.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:46.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:04:50.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:15.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:17.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:19.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:42.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:44.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:05:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:16.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:26.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:47.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:49.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:06:51.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:18.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:20.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:41.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:43.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:07:45.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:14.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:16.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:26.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:44.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:48.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:08:50.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:13.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:17.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:19.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:40.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:09:46.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:14.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:16.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:26.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:44.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:48.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:10:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:15.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:17.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:19.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:40.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:44.712] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:11:46.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:14.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:16.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:27.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:49.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:12:51.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:15.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:17.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:19.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:42.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:44.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:13:46.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:14.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:16.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:26.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:44.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:48.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:14:50.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:13.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:17.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:19.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:42.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:44.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:15:46.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:14.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:16.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:26.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:46.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:48.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:16:50.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:14.044] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:18.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:20.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:41.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:43.748] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:17:45.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:18:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:15.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:17.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:19:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:46.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:20:50.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:17.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:21:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:14.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:26.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:45.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:49.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:22:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:14.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:20.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:41.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:43.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:23:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:14.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:19.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:26.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:46.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:48.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:24:50.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:13.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:17.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:19.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:42.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:44.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:25:46.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:12.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:16.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:26.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:46.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:26:50.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:14.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:16.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:18.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:27:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:45.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:47.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:28:49.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:12.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:14.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:16.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:45.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:49.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:29:51.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:32.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:34.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:36.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:41.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:43.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:30:45.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:14.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:23.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:25.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:43.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:47.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:31:49.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:11.489] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:15.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:48.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:50.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:32:52.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:33:19.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:33:36.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:33:38.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:14.708] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:19.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:26.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:46.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:48.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:34:50.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:16.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:18.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:20.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:35:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:36:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:14.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:16.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:18.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:42.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:44.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:37:46.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:38:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:17.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:39:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:45.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:40:51.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:16.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:18.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:41.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:43.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:41:45.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:12.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:16.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:48.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:42:50.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:13.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:17.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:19.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:42.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:44.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:43:46.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:44.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:48.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:44:50.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:13.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:17.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:19.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:42.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:44.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:45:46.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:14.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:23.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:25.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:44.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:48.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:46:50.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:13.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:17.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:19.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:42.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:44.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:47:46.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:12.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:16.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:26.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:44.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:48.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:48:50.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:15.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:17.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:19.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:42.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:44.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:49:46.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:14.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:16.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:26.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:45.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:47.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:50:49.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:13.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:17.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:41.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:43.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:51:45.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:12.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:15.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:25.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:43.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:47.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:52:49.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:11.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:15.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:17.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:46.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:49.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:53:51.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:31.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:33.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:35.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:42.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:44.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:54:46.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:14.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:19.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:26.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:46.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:48.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:55:50.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:15.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:17.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:19.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:42.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:44.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:56:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:14.711] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:16.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:26.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:46.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:48.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:57:50.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:16.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:18.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:20.053] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:58:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:15.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:25.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:43.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:47.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T07:59:49.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:11.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:15.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:17.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:48.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:50.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:00:52.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:01:29.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:01:36.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:01:38.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:12.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:16.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:46.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:02:50.802] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:15.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:19.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:42.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:03:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:14.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:19.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:26.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:45.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:49.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:04:51.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:18.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:20.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:41.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:43.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:05:45.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:12.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:16.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:26.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:44.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:48.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:06:50.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:13.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:17.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:19.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:42.326] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:44.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:07:46.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:12.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:16.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:26.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:44.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:08:50.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:14.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:16.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:18.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:09:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:10:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:41.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:11:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:14.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:45.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:47.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:12:49.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:13.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:15.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:48.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:50.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:13:52.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:20.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:36.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:38.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:40.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:44.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:14:46.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:14.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:16.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:26.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:46.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:15:50.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:13.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:42.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:16:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:14.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:16.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:26.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:45.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:49.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:17:51.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:18.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:20.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:41.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:43.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:18:45.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:14.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:16.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:26.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:46.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:48.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:19:50.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:13.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:17.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:19.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:42.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:44.326] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:20:46.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:12.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:16.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:26.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:45.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:47.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:21:49.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:11.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:15.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:49.079] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:51.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:22:53.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:17.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:35.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:37.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:42.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:44.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:23:46.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:12.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:16.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:26.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:46.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:48.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:24:50.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:16.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:18.047] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:20.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:41.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:25:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:44.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:26:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:19.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:27:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:28:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:15.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:17.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:19.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:42.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:44.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:29:46.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:14.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:19.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:26.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:47.118] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:49.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:30:51.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:20.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:31:45.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:14.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:16.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:26.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:46.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:48.422] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:32:50.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:13.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:17.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:19.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:40.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:44.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:33:46.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:12.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:16.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:26.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:46.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:48.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:34:50.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:15.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:17.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:19.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:42.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:44.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:35:46.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:12.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:16.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:26.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:47.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:49.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:36:51.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:16.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:18.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:20.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:41.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:43.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:37:45.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:13.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:15.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:25.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:45.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:47.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:38:49.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:13.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:17.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:48.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:50.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:39:52.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:30.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:35.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:37.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:39.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:44.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:40:46.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:14.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:16.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:26.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:46.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:48.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:41:50.517] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:13.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:17.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:19.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:42.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:44.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:42:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:14.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:19.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:26.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:44.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:47.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:43:49.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:11.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:16.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:18.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:41.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:43.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:44:45.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:14.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:19.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:26.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:45.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:47.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:45:49.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:11.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:15.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:17.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:48.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:50.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:46:52.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:47:15.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:47:36.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:47:38.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:12.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:16.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:26.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:44.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:48.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:48:50.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:13.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:18.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:20.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:49:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:50:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:40.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:51:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:12.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:46.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:48.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:52:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:15.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:53:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:45.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:54:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:18.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:55:45.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:14.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:16.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:56:50.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:13.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:17.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:40.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:57:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:12.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:16.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:26.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:46.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:48.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:58:50.815] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:15.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:19.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:40.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:44.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T08:59:46.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:12.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:16.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:26.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:47.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:49.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:00:51.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:14.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:18.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:20.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:41.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:43.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:01:45.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:12.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:26.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:44.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:48.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:02:50.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:15.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:17.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:19.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:42.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:44.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:03:46.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:14.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:24.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:26.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:46.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:48.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:04:50.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:13.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:16.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:18.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:41.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:43.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:05:45.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:44.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:06:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:15.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:17.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:42.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:44.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:07:46.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:14.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:16.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:26.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:46.624] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:48.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:08:50.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:13.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:17.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:19.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:42.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:44.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:09:46.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:14.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:16.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:26.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:45.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:48.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:10:50.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:15.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:17.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:19.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:39.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:43.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:11:45.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:14.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:24.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:26.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:46.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:48.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:12:50.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:13.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:17.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:19.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:42.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:44.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:13:46.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:14.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:26.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:48.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:14:50.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:13.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:17.908] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:19.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:42.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:44.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:15:46.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:14.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:16.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:26.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:46.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:48.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:16:50.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:13.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:17.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:19.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:40.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:44.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:17:46.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:14.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:19.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:26.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:44.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:48.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:18:50.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:15.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:17.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:19.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:42.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:44.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:19:46.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:12.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:16.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:26.748] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:45.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:47.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:20:49.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:13.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:17.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:19.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:41.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:43.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:21:45.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:12.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:18.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:25.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:43.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:47.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:22:49.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:12.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:16.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:18.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:42.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:44.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:23:46.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:12.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:16.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:26.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:46.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:48.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:24:50.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:15.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:17.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:42.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:44.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:25:46.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:14.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:16.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:26.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:45.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:49.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:26:51.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:18.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:20.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:41.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:43.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:27:45.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:12.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:16.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:26.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:46.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:48.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:28:50.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:13.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:17.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:19.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:42.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:44.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:29:46.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:12.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:16.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:26.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:44.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:48.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:30:50.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:15.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:19.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:40.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:31:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:12.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:27.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:47.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:49.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:32:51.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:16.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:18.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:20.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:41.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:43.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:33:46.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:14.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:16.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:26.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:44.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:48.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:34:50.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:13.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:17.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:19.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:42.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:44.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:35:46.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:12.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:16.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:26.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:46.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:48.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:36:50.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:15.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:17.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:19.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:37:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:38:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:15.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:17.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:41.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:39:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:12.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:16.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:40:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:40.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:41:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:12.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:16.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:26.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:46.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:48.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:42:50.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:13.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:17.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:19.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:40.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:44.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:43:46.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:12.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:16.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:26.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:46.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:48.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:44:50.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:13.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:17.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:19.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:40.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:45:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:12.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:16.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:26.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:44.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:48.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:46:50.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:15.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:18.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:20.039] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:47:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:19.076] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:46.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:48:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:40.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:49:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:50:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:15.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:17.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:40.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:43.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:51:45.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:13.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:17.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:27.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:47.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:49.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:52:51.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:14.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:18.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:20.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:42.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:44.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:53:46.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:14.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:19.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:26.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:46.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:48.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:54:50.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:13.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:17.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:19.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:42.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:44.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:55:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:14.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:16.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:26.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:46.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:48.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:56:50.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:15.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:18.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:20.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:57:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:14.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:16.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:26.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:58:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:13.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:17.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:19.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T09:59:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:14.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:00:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:17.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:42.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:01:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:12.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:47.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:49.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:02:51.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:16.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:18.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:03:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:12.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:16.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:04:50.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:15.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:42.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:44.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:05:46.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:14.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:19.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:26.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:44.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:48.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:06:50.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:13.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:17.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:42.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:44.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:07:46.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:14.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:16.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:26.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:45.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:49.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:08:51.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:14.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:18.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:20.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:39.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:43.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:09:45.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:14.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:16.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:26.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:46.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:48.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:10:50.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:15.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:17.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:40.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:44.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:11:46.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:12.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:16.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:26.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:46.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:48.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:12:50.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:15.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:17.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:19.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:42.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:13:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:12.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:16.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:27.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:46.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:48.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:14:50.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:13.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:17.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:19.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:42.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:44.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:15:46.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:12.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:16.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:26.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:46.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:48.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:16:50.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:13.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:17.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:19.711] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:42.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:44.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:17:46.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:14.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:19.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:26.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:46.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:48.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:18:50.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:16.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:18.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:20.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:41.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:43.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:19:45.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:44.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:20:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:21:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:46.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:22:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:13.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:17.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:19.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:23:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:12.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:16.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:45.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:24:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:20.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:41.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:43.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:25:45.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:14.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:16.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:26.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:46.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:48.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:26:50.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:13.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:17.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:19.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:40.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:27:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:14.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:19.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:45.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:47.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:28:49.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:14.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:16.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:18.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:29:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:46.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:30:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:31:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:12.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:44.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:32:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:42.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:33:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:45.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:48.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:34:50.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:13.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:17.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:19.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:40.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:44.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:35:46.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:12.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:16.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:26.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:46.489] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:48.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:36:50.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:15.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:17.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:19.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:42.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:44.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:37:46.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:14.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:16.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:26.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:44.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:48.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:38:50.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:13.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:18.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:20.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:39:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:44.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:40:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:41:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:44.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:42:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:40.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:44.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:43:46.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:13.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:15.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:25.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:44.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:44:50.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:15.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:17.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:19.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:40.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:44.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:45:46.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:12.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:16.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:26.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:44.482] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:48.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:46:50.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:13.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:17.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:19.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:42.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:44.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:47:46.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:14.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:19.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:26.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:44.859] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:48.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:48:50.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:13.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:18.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:20.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:49:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:46.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:50:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:13.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:51:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:52:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:13.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:17.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:40.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:44.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:53:46.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:12.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:16.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:26.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:45.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:49.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:54:51.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:16.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:18.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:20.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:39.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:43.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:55:45.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:12.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:26.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:44.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:48.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:56:50.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:13.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:17.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:19.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:40.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:44.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:57:46.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:12.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:16.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:26.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:46.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:48.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:58:50.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:13.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:17.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:19.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:40.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:44.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T10:59:46.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:14.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:24.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:26.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:47.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:00:51.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:14.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:18.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:20.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:41.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:43.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:01:45.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:14.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:16.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:26.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:46.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:48.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:02:50.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:13.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:17.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:19.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:40.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:44.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:03:46.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:14.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:19.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:26.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:44.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:48.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:04:50.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:15.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:17.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:19.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:05:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:06:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:41.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:07:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:12.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:15.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:25.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:43.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:47.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:08:49.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:11.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:17.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:47.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:51.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:09:53.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:31.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:35.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:37.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:42.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:44.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:10:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:12.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:16.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:26.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:46.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:48.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:11:50.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:13.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:18.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:20.023] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:12:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:12.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:44.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:13:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:17.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:19.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:42.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:44.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:14:46.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:12.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:16.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:26.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:46.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:48.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:15:50.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:13.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:17.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:19.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:40.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:44.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:16:46.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:14.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:16.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:45.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:49.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:17:51.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:16.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:18.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:20.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:41.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:43.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:18:45.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:12.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:16.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:26.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:46.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:48.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:19:50.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:15.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:17.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:19.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:42.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:44.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:20:46.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:12.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:16.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:26.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:46.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:48.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:21:50.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:13.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:16.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:18.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:42.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:44.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:22:46.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:16.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:26.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:47.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:49.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:23:51.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:16.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:18.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:20.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:40.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:42.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:24:44.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:14.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:16.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:26.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:46.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:48.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:25:50.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:13.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:17.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:19.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:42.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:44.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:26:46.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:12.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:16.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:26.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:46.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:48.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:27:50.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:16.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:18.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:20.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:28:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:44.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:29:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:30:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:12.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:16.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:26.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:44.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:48.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:31:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:13.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:17.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:42.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:44.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:32:46.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:12.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:16.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:26.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:47.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:49.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:33:51.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:16.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:18.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:20.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:39.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:43.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:34:45.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:12.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:15.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:25.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:43.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:47.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:35:49.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:12.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:16.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:18.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:42.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:44.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:36:46.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:12.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:16.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:26.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:46.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:48.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:37:50.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:15.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:42.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:44.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:38:46.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:14.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:24.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:26.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:45.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:48.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:39:50.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:15.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:17.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:19.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:42.044] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:44.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:40:46.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:12.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:16.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:26.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:46.512] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:48.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:41:50.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:13.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:17.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:19.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:40.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:44.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:42:46.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:12.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:16.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:26.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:46.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:48.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:43:50.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:16.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:18.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:20.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:44:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:14.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:19.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:46.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:48.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:45:50.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:19.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:41.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:43.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:46:45.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:14.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:16.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:26.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:48.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:47:50.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:13.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:17.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:19.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:40.626] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:44.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:48:46.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:14.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:19.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:26.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:45.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:49.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:49:51.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:18.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:20.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:41.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:43.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:50:45.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:12.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:16.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:26.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:46.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:48.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:51:50.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:14.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:16.525] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:18.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:40.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:43.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:52:45.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:14.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:16.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:26.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:44.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:48.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:53:50.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:18.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:20.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:41.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:43.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:54:45.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:55:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:15.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:17.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:44.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:56:46.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:14.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:20.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:25.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:45.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:47.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:57:49.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:11.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:15.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:17.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:46.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:50.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:58:52.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:20.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:34.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:36.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:40.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:44.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T11:59:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:14.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:19.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:26.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:46.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:48.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:00:50.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:13.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:18.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:20.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:01:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:14.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:19.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:26.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:46.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:02:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:15.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:17.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:03:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:14.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:19.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:26.471] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:44.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:48.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:04:50.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:15.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:17.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:42.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:44.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:05:46.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:14.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:16.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:26.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:47.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:49.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:06:51.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:14.126] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:18.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:20.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:41.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:43.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:07:45.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:12.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:15.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:25.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:44.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:46.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:08:48.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:13.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:15.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:17.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:48.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:50.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:09:52.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:17.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:35.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:37.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:39.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:43.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:10:45.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:12.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:15.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:25.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:45.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:47.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:11:49.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:11.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:15.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:17.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:48.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:50.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:12:52.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:13:14.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:13:37.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:13:39.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:12.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:44.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:14:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:15.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:17.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:15:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:14.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:26.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:47.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:49.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:16:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:16.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:18.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:41.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:43.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:17:45.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:12.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:16.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:26.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:44.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:18:50.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:13.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:17.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:40.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:19:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:12.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:16.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:26.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:46.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:48.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:20:50.815] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:15.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:17.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:42.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:44.690] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:21:46.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:12.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:16.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:26.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:45.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:22:51.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:14.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:18.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:20.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:39.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:43.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:23:45.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:12.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:16.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:26.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:44.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:48.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:24:50.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:13.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:17.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:19.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:40.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:44.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:25:46.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:12.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:16.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:26.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:46.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:48.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:26:50.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:13.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:17.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:19.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:27:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:28:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:40.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:29:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:14.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:30:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:13.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:31:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:13.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:15.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:25.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:46.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:48.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:32:50.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:15.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:17.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:19.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:42.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:44.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:33:46.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:12.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:16.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:26.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:44.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:48.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:34:50.530] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:13.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:17.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:19.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:42.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:44.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:35:46.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:12.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:16.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:26.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:46.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:48.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:36:50.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:13.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:18.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:20.031] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:37:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:48.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:38:50.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:39:45.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:14.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:40:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:15.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:41:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:14.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:26.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:45.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:49.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:42:51.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:16.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:20.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:39.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:43.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:43:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:12.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:16.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:26.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:44.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:48.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:44:50.433] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:13.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:19.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:40.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:44.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:45:46.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:14.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:16.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:26.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:46.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:48.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:46:50.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:15.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:17.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:19.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:40.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:44.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:47:46.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:14.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:16.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:25.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:44.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:48.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:48:50.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:15.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:17.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:19.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:42.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:44.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:49:46.079] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:12.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:16.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:26.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:44.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:48.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:50:50.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:13.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:17.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:19.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:42.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:44.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:51:46.462] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:14.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:16.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:26.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:46.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:48.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:52:50.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:16.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:18.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:20.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:41.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:43.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:53:45.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:44.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:54:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:41.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:55:45.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:15.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:25.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:43.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:47.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:56:49.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:11.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:15.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:48.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:50.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:57:52.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:17.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:34.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:36.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:42.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:44.482] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:58:46.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:12.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:16.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:26.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:46.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:48.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T12:59:50.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:16.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:18.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:20.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:41.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:43.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:00:45.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:12.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:16.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:26.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:47.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:01:49.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:14.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:16.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:18.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:40.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:44.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:02:46.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:12.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:16.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:26.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:44.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:48.746] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:03:50.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:13.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:17.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:19.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:40.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:44.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:04:46.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:14.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:16.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:26.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:47.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:49.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:05:51.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:16.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:18.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:20.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:41.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:43.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:06:45.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:14.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:16.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:26.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:46.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:48.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:07:50.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:13.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:17.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:19.530] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:42.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:44.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:08:46.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:14.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:16.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:26.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:46.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:48.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:09:50.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:13.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:17.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:19.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:42.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:44.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:10:46.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:14.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:16.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:26.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:45.124] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:49.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:11:51.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:13.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:17.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:19.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:42.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:44.037] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:12:46.045] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:12.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:16.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:26.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:46.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:48.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:13:50.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:15.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:17.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:19.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:42.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:44.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:14:46.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:14.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:16.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:26.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:46.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:48.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:15:50.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:15.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:18.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:20.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:16:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:17:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:12.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:16.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:18.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:40.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:44.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:18:46.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:48.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:19:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:17.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:19.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:42.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:20:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:14.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:26.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:45.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:49.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:21:51.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:14.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:20.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:41.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:43.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:22:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:14.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:16.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:48.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:23:50.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:13.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:17.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:19.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:42.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:44.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:24:46.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:12.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:16.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:26.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:25:50.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:15.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:17.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:19.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:42.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:44.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:26:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:16.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:26.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:47.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:49.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:27:51.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:16.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:18.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:20.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:39.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:43.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:28:45.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:14.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:24.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:26.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:46.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:48.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:29:50.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:13.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:17.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:19.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:40.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:44.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:30:46.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:14.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:16.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:26.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:46.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:48.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:31:50.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:13.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:17.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:19.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:32:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:15.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:20.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:27.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:47.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:49.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:33:51.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:16.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:18.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:20.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:39.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:44.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:34:46.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:14.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:16.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:26.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:48.505] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:35:50.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:15.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:17.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:19.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:40.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:44.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:36:46.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:12.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:16.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:26.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:46.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:48.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:37:50.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:13.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:17.993] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:20.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:38:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:44.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:39:50.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:19.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:42.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:44.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:40:46.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:12.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:16.462] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:26.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:46.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:48.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:41:50.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:15.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:17.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:19.802] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:42.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:44.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:42:46.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:14.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:16.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:26.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:45.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:49.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:43:51.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:16.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:18.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:20.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:39.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:43.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:44:45.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:12.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:16.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:26.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:46.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:48.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:45:50.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:13.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:17.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:19.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:42.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:44.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:46:46.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:12.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:16.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:26.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:45.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:47.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:47:49.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:13.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:15.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:17.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:46.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:50.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:48:53.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:32.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:34.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:36.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:40.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:44.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:49:46.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:12.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:16.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:26.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:46.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:48.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:50:50.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:16.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:18.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:20.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:51:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:52:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:19.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:40.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:44.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:53:46.180] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:12.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:16.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:26.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:44.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:47.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:54:49.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:13.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:15.787] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:17.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:46.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:49.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:55:51.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:23.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:35.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:37.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:40.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:44.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:56:46.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:14.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:16.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:26.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:44.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:57:50.802] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:13.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:17.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:42.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:44.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:58:46.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:14.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:19.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:26.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:45.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:48.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T13:59:50.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:13.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:17.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:19.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:42.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:44.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:00:46.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:12.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:16.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:26.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:44.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:48.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:01:50.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:13.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:17.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:19.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:42.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:44.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:02:46.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:14.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:16.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:26.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:46.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:48.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:03:50.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:14.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:18.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:20.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:04:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:05:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:13.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:17.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:40.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:43.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:06:45.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:12.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:16.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:26.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:07:50.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:13.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:17.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:08:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:12.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:47.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:09:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:14.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:39.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:43.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:10:45.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:12.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:16.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:26.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:46.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:48.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:11:50.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:13.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:17.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:19.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:42.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:44.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:12:46.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:12.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:16.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:26.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:46.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:48.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:13:50.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:13.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:17.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:19.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:42.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:44.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:14:46.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:12.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:16.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:26.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:47.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:15:51.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:16.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:18.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:20.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:41.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:43.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:16:45.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:12.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:16.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:26.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:44.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:48.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:17:50.475] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:13.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:17.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:19.596] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:40.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:44.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:18:46.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:14.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:16.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:26.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:46.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:48.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:19:50.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:15.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:17.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:19.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:20:46.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:13.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:17.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:27.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:47.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:49.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:21:51.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:18.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:20.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:41.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:44.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:22:46.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:12.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:16.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:26.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:46.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:48.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:23:50.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:13.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:17.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:19.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:42.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:24:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:12.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:16.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:26.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:46.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:48.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:25:50.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:15.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:17.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:20.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:41.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:26:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:15.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:25.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:43.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:47.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:27:49.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:11.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:15.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:17.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:46.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:50.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:28:52.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:32.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:34.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:36.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:39.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:43.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:29:45.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:14.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:16.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:26.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:46.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:48.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:30:50.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:14.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:16.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:18.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:42.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:44.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:31:46.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:12.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:16.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:26.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:46.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:48.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:32:50.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:15.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:18.020] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:20.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:33:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:12.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:16.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:26.081] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:34:50.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:12.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:16.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:18.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:42.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:44.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:35:46.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:14.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:19.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:26.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:45.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:47.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:36:49.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:11.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:15.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:17.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:48.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:50.037] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:37:52.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:31.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:33.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:35.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:42.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:44.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:38:46.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:14.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:16.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:26.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:45.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:48.085] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:39:50.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:13.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:17.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:19.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:41.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:43.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:40:45.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:12.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:16.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:26.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:44.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:48.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:41:50.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:12.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:18.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:42.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:44.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:42:46.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:14.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:16.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:26.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:46.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:48.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:43:50.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:14.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:18.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:20.081] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:41.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:43.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:44:45.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:46.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:48.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:45:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:13.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:17.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:19.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:42.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:44.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:46:46.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:12.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:16.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:26.464] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:46.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:48.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:47:50.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:15.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:17.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:19.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:40.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:44.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:48:46.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:12.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:16.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:26.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:44.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:49.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:49:51.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:16.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:18.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:20.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:39.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:43.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:50:45.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:12.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:16.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:26.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:44.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:48.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:51:50.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:12.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:16.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:18.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:40.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:44.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:52:46.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:12.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:16.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:26.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:46.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:48.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:53:50.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:13.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:17.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:19.904] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:42.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:44.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:54:46.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:12.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:16.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:26.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:47.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:55:51.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:16.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:18.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:20.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:41.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:43.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:56:45.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:12.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:26.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:46.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:48.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:57:50.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:15.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:17.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:19.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:42.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:44.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:58:46.326] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:12.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:16.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:26.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:46.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:48.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T14:59:50.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:15.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:17.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:19.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:40.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:44.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:00:46.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:14.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:16.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:26.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:45.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:49.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:01:51.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:14.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:18.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:20.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:41.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:43.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:02:45.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:12.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:16.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:26.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:46.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:48.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:03:50.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:13.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:17.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:19.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:42.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:44.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:04:46.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:12.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:16.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:26.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:46.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:48.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:05:50.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:15.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:17.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:19.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:06:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:07:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:15.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:17.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:41.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:43.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:08:45.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:14.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:16.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:48.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:09:50.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:15.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:17.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:10:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:26.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:47.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:11:51.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:16.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:18.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:20.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:39.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:43.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:12:45.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:13.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:18.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:25.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:45.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:47.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:13:49.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:13.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:15.530] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:48.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:50.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:14:52.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:17.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:35.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:37.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:39.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:43.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:15:45.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:14.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:19.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:26.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:44.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:48.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:16:50.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:13.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:17.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:19.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:40.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:17:46.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:14.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:16.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:26.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:44.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:48.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:18:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:16.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:18.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:19:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:46.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:20:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:43.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:21:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:14.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:44.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:22:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:13.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:19.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:40.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:23:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:14.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:26.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:49.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:24:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:18.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:20.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:41.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:43.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:25:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:12.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:26.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:46.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:48.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:26:50.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:13.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:17.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:19.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:40.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:44.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:27:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:12.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:16.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:26.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:44.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:48.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:28:50.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:15.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:17.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:19.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:42.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:44.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:29:46.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:14.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:16.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:27.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:45.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:49.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:30:51.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:16.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:18.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:20.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:39.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:31:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:12.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:16.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:26.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:46.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:48.473] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:32:50.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:13.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:17.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:19.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:42.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:44.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:33:46.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:12.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:16.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:26.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:44.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:48.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:34:50.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:13.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:17.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:19.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:35:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:44.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:36:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:15.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:17.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:37:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:44.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:38:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:13.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:17.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:42.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:44.646] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:39:46.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:12.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:16.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:47.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:49.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:40:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:16.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:18.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:41:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:14.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:19.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:26.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:44.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:48.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:42:50.434] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:13.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:17.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:19.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:42.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:44.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:43:46.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:12.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:16.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:26.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:46.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:48.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:44:50.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:13.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:17.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:19.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:42.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:44.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:45:46.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:14.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:16.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:26.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:45.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:49.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:46:51.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:16.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:18.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:20.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:39.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:43.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:47:45.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:14.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:16.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:26.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:46.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:48.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:48:50.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:15.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:17.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:19.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:42.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:44.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:49:46.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:12.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:16.624] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:26.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:44.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:48.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:50:50.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:13.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:16.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:18.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:51:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:46.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:52:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:17.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:41.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:53:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:14.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:47.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:54:49.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:13.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:17.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:48.024] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:50.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:55:52.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:17.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:34.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:36.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:40.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:44.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:56:46.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:14.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:16.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:26.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:46.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:48.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:57:50.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:16.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:18.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:20.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:41.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:43.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:58:45.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:16.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:26.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T15:59:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:15.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:17.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:42.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:44.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:00:46.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:14.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:16.456] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:26.476] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:46.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:48.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:01:50.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:13.746] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:17.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:19.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:40.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:44.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:02:46.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:12.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:16.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:47.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:49.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:03:51.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:16.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:18.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:20.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:39.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:43.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:04:45.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:12.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:18.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:25.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:43.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:47.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:05:49.337] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:11.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:15.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:17.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:46.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:50.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:06:52.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:33.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:35.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:37.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:39.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:43.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:07:45.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:12.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:16.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:26.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:46.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:48.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:08:50.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:13.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:17.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:19.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:42.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:44.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:09:46.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:12.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:16.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:26.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:46.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:48.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:10:50.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:13.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:17.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:19.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:42.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:44.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:11:46.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:14.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:16.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:26.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:45.081] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:49.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:12:51.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:16.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:18.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:20.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:41.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:43.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:13:45.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:12.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:16.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:26.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:44.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:48.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:14:50.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:13.473] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:17.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:19.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:40.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:44.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:15:46.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:12.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:16.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:26.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:44.744] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:48.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:16:50.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:15.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:19.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:40.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:44.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:17:46.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:14.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:16.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:26.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:47.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:49.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:18:51.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:16.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:18.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:20.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:41.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:43.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:19:45.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:12.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:26.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:48.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:20:50.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:13.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:17.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:19.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:42.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:44.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:21:46.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:14.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:16.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:26.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:44.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:48.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:22:50.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:14.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:16.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:18.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:23:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:12.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:16.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:26.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:44.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:24:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:17.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:41.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:25:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:44.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:26:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:13.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:17.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:27:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:12.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:16.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:26.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:45.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:28:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:16.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:18.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:20.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:41.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:43.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:29:45.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:12.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:16.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:26.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:44.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:48.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:30:50.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:14.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:16.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:18.532] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:42.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:31:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:14.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:16.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:46.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:48.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:32:50.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:13.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:17.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:33:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:46.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:34:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:19.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:35:45.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:12.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:46.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:48.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:36:50.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:13.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:19.883] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:40.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:44.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:37:46.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:14.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:19.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:26.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:47.108] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:49.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:38:51.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:16.221] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:20.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:40.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:42.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:39:44.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:14.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:19.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:26.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:46.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:48.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:40:50.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:15.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:17.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:19.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:42.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:44.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:41:46.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:14.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:16.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:26.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:46.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:48.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:42:50.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:13.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:18.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:20.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:43:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:12.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:16.067] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:48.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:44:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:13.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:17.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:19.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:41.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:43.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:45:45.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:14.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:19.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:26.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:46.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:48.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:46:50.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:13.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:17.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:19.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:42.626] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:44.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:47:46.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:12.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:16.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:26.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:45.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:49.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:48:51.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:16.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:18.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:20.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:41.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:43.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:49:45.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:12.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:16.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:26.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:46.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:48.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:50:50.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:13.484] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:17.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:19.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:42.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:44.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:51:46.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:12.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:16.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:26.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:44.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:48.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:52:50.794] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:13.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:17.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:19.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:40.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:44.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:53:46.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:14.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:16.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:26.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:46.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:48.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:54:50.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:13.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:17.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:19.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:42.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:44.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:55:46.058] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:12.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:16.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:26.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:44.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:48.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:56:50.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:15.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:17.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:19.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:42.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:44.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:57:46.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:12.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:16.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:26.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:44.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:48.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:58:50.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:15.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:17.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:19.054] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:41.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:43.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T16:59:45.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:12.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:16.093] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:26.109] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:46.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:48.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:00:50.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:13.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:17.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:19.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:41.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:43.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:01:45.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:12.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:16.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:26.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:46.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:48.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:02:50.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:13.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:17.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:19.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:40.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:44.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:03:46.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:14.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:16.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:26.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:47.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:49.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:04:51.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:16.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:18.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:20.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:41.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:43.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:05:45.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:12.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:16.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:26.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:46.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:48.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:06:50.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:15.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:17.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:19.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:40.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:44.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:07:46.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:12.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:16.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:26.623] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:44.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:48.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:08:50.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:15.900] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:17.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:19.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:42.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:44.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:09:46.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:14.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:16.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:27.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:47.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:49.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:10:51.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:14.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:18.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:20.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:41.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:43.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:11:45.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:14.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:16.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:26.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:46.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:48.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:12:50.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:15.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:17.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:19.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:42.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:44.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:13:46.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:14.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:16.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:26.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:44.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:48.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:14:50.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:15.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:17.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:19.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:15:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:15.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:17.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:27.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:45.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:49.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:16:51.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:16.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:18.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:20.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:41.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:44.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:17:46.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:14.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:19.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:26.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:44.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:48.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:18:50.505] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:15.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:17.616] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:19.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:40.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:44.381] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:19:46.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:12.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:15.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:25.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:45.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:47.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:20:49.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:11.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:15.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:18.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:39.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:43.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:21:45.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:14.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:18.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:25.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:43.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:46.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:22:48.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:11.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:15.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:17.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:48.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:50.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:23:52.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:24:27.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:24:36.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:24:38.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:12.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:16.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:26.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:45.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:47.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:25:49.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:14.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:16.983] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:18.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:41.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:43.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:26:45.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:19.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:46.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:27:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:17.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:41.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:28:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:14.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:48.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:29:50.772] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:15.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:42.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:30:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:26.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:45.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:49.132] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:31:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:14.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:18.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:20.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:41.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:43.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:32:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:12.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:16.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:26.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:48.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:33:50.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:15.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:17.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:19.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:40.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:44.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:34:46.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:14.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:19.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:26.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:46.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:48.789] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:35:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:15.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:17.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:19.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:42.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:44.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:36:46.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:14.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:16.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:26.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:45.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:49.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:37:51.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:16.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:18.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:20.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:41.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:43.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:38:45.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:14.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:16.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:26.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:44.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:48.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:39:50.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:15.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:17.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:19.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:40.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:44.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:40:46.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:14.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:16.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:26.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:44.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:48.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:41:50.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:13.905] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:15.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:17.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:47.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:51.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:42:53.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:43:18.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:43:37.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:43:39.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:46.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:44:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:15.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:17.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:19.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:42.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:44.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:45:46.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:14.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:19.449] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:26.464] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:46.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:48.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:46:50.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:13.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:17.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:19.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:42.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:44.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:47:46.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:14.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:16.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:26.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:45.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:49.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:48:51.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:14.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:18.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:20.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:39.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:43.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:49:45.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:13.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:15.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:25.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:43.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:47.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:50:49.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:11.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:15.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:17.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:48.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:50.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:51:52.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:52:17.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:52:36.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:52:38.814] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:14.610] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:16.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:26.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:44.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:53:50.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:15.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:17.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:19.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:42.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:44.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:54:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:12.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:27.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:46.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:48.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:55:50.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:13.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:17.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:19.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:41.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:43.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:56:45.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:12.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:16.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:26.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:44.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:48.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:57:50.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:15.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:17.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:19.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:42.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:44.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:58:46.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:14.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:16.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:26.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:45.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:49.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T17:59:51.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:16.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:18.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:20.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:41.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:43.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:00:45.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:12.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:15.138] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:25.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:43.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:47.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:01:49.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:11.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:15.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:17.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:46.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:49.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:02:51.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:19.787] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:33.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:35.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:42.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:44.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:03:46.078] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:14.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:24.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:26.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:46.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:48.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:04:50.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:13.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:17.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:19.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:42.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:44.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:05:46.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:14.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:16.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:26.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:46.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:48.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:06:50.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:16.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:18.046] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:20.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:07:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:08:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:15.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:42.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:09:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:12.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:16.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:26.468] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:46.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:48.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:10:50.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:13.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:17.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:42.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:44.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:11:46.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:14.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:16.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:26.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:45.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:49.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:12:51.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:14.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:18.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:20.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:41.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:43.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:13:45.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:12.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:16.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:26.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:44.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:48.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:14:50.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:13.401] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:17.427] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:19.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:41.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:43.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:15:45.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:12.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:16.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:26.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:46.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:48.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:16:50.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:13.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:17.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:19.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:42.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:44.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:17:46.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:14.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:16.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:26.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:45.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:49.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:18:51.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:16.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:18.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:20.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:41.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:43.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:19:45.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:12.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:16.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:26.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:44.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:48.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:20:50.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:13.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:17.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:19.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:42.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:44.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:21:46.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:12.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:16.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:26.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:44.796] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:48.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:22:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:15.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:17.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:19.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:40.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:44.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:23:46.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:14.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:16.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:27.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:47.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:49.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:24:51.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:14.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:18.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:20.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:41.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:43.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:25:45.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:12.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:16.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:26.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:46.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:48.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:26:50.495] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:13.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:17.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:19.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:40.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:44.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:27:46.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:14.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:19.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:26.674] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:44.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:48.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:28:50.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:15.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:17.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:19.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:29:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:14.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:19.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:26.083] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:46.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:48.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:30:50.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:13.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:17.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:19.400] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:41.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:43.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:31:45.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:14.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:16.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:26.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:46.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:48.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:32:50.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:13.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:17.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:19.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:40.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:44.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:33:46.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:12.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:16.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:26.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:45.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:49.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:34:51.140] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:18.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:20.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:39.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:43.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:35:45.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:12.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:16.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:26.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:46.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:48.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:36:50.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:13.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:17.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:19.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:42.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:44.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:37:46.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:12.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:16.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:26.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:44.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:48.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:38:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:13.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:17.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:19.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:40.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:44.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:39:46.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:14.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:16.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:26.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:45.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:49.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:40:51.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:14.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:18.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:41.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:43.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:41:45.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:13.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:18.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:25.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:43.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:47.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:42:49.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:14.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:16.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:18.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:40.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:44.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:43:46.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:12.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:16.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:26.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:46.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:48.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:44:50.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:16.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:18.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:20.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:45:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:16.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:44.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:46:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:15.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:19.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:40.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:43.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:47:45.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:12.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:16.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:26.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:48.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:48:50.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:13.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:17.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:19.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:40.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:44.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:49:46.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:12.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:16.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:26.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:47.121] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:50:51.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:16.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:18.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:20.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:41.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:43.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:51:45.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:12.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:16.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:26.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:44.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:48.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:52:50.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:13.513] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:17.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:19.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:40.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:44.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:53:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:12.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:16.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:26.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:44.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:48.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:54:50.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:13.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:17.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:19.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:40.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:44.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:55:46.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:14.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:16.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:26.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:47.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:49.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:56:51.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:16.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:18.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:20.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:39.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:57:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:12.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:16.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:26.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:46.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:48.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:58:50.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:13.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:17.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:19.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:42.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:44.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T18:59:46.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:12.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:16.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:26.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:46.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:48.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:00:50.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:12.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:16.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:18.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:41.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:43.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:01:45.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:19.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:44.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:02:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:40.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:03:45.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:12.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:46.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:04:50.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:13.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:17.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:42.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:44.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:05:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:12.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:46.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:06:50.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:15.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:17.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:19.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:40.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:44.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:07:46.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:12.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:16.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:26.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:46.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:48.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:08:50.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:13.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:17.637] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:19.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:40.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:44.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:09:46.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:14.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:16.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:26.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:44.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:48.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:10:50.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:16.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:18.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:20.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:11:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:19.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:12:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:13.369] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:17.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:42.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:44.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:13:46.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:12.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:16.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:26.467] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:44.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:48.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:14:50.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:13.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:17.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:19.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:42.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:44.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:15:46.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:12.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:16.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:26.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:47.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:49.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:16:51.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:16.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:18.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:20.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:39.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:42.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:17:44.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:13.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:15.214] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:25.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:43.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:47.422] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:18:49.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:13.507] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:15.531] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:17.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:48.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:50.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:19:52.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:26.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:33.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:35.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:41.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:43.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:20:45.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:14.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:16.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:26.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:45.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:47.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:21:49.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:11.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:15.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:47.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:50.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:22:52.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:17.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:34.224] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:36.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:42.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:23:46.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:14.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:19.764] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:26.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:44.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:48.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:24:50.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:16.053] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:18.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:20.098] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:41.745] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:43.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:25:45.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:26:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:13.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:41.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:27:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:14.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:46.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:28:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:13.831] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:17.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:19.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:42.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:29:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:26.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:47.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:30:51.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:16.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:18.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:20.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:31:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:14.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:16.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:46.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:48.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:32:50.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:15.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:17.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:19.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:42.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:44.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:33:46.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:26.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:46.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:48.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:34:50.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:13.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:17.906] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:19.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:40.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:44.686] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:35:46.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:14.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:18.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:25.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:45.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:47.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:36:49.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:14.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:16.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:18.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:37:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:14.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:19.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:26.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:48.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:38:50.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:15.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:19.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:40.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:44.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:39:46.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:14.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:18.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:25.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:46.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:40:50.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:13.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:17.263] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:19.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:42.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:44.029] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:41:46.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:12.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:16.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:26.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:46.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:48.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:42:50.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:13.608] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:17.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:19.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:42.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:44.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:43:46.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:12.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:16.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:26.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:44.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:48.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:44:50.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:15.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:18.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:20.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:45:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:45.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:47.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:46:49.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:11.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:15.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:46.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:50.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:47:52.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:24.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:35.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:37.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:39.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:43.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:48:45.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:12.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:16.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:26.124] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:44.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:48.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:49:50.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:13.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:17.431] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:19.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:42.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:44.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:50:46.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:12.476] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:16.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:26.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:46.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:48.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:51:50.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:15.787] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:17.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:19.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:40.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:44.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:52:46.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:12.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:16.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:26.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:47.049] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:49.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:53:51.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:14.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:18.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:20.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:39.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:43.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:54:45.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:14.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:16.148] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:26.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:46.328] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:48.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:55:50.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:12.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:16.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:18.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:42.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:44.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:56:46.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:12.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:16.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:26.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:44.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:48.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:57:50.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:13.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:17.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:19.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:40.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:44.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:58:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:15.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:17.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:27.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:47.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:49.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T19:59:51.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:14.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:18.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:20.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:40.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:44.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:00:46.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:12.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:16.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:26.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:44.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:48.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:01:50.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:15.600] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:17.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:19.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:40.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:44.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:02:46.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:12.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:16.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:26.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:44.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:48.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:03:50.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:13.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:18.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:20.021] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:04:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:12.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:15.069] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:25.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:43.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:47.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:05:49.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:13.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:15.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:46.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:50.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:06:52.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:07:34.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:07:36.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:07:38.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:12.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:16.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:26.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:46.757] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:48.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:08:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:13.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:17.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:19.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:40.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:44.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:09:46.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:12.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:16.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:26.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:45.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:49.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:10:51.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:14.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:18.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:20.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:41.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:43.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:11:45.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:14.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:24.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:26.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:48.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:12:50.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:13.516] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:17.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:19.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:40.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:44.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:13:46.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:12.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:16.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:26.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:44.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:14:50.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:15.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:17.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:19.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:40.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:44.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:15:46.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:12.988] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:27.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:49.206] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:16:51.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:14.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:17.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:19.336] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:40.082] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:44.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:17:46.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:12.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:16.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:26.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:45.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:47.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:18:49.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:12.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:16.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:18.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:42.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:44.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:19:46.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:14.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:19.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:26.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:47.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:49.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:20:51.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:16.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:18.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:20.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:41.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:43.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:21:45.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:13.119] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:15.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:25.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:43.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:47.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:22:49.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:12.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:16.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:18.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:40.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:44.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:23:46.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:14.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:16.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:26.621] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:46.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:48.815] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:24:50.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:15.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:17.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:19.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:42.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:44.697] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:25:46.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:14.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:16.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:26.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:47.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:49.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:26:51.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:16.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:18.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:20.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:39.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:43.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:27:45.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:12.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:16.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:26.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:46.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:48.462] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:28:50.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:13.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:17.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:19.596] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:42.344] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:44.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:29:46.361] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:12.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:16.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:26.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:46.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:48.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:30:50.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:15.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:17.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:19.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:42.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:44.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:31:46.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:13.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:17.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:27.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:45.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:49.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:32:51.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:18.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:20.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:41.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:44.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:33:46.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:12.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:16.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:26.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:46.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:48.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:34:50.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:17.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:19.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:40.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:35:46.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:14.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:16.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:26.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:46.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:48.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:36:50.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:15.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:17.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:20.006] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:37:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:47.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:38:49.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:11.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:15.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:17.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:46.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:50.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:39:52.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:29.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:34.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:36.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:41.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:43.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:40:45.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:14.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:19.196] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:26.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:44.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:48.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:41:50.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:14.492] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:16.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:18.528] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:40.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:44.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:42:46.393] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:14.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:16.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:44.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:48.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:43:50.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:15.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:17.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:44:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:44.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:45:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:17.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:41.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:43.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:46:45.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:26.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:46.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:48.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:47:50.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:17.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:40.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:48:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:12.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:16.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:47.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:49.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:49:51.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:14.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:18.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:20.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:39.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:43.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:50:45.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:12.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:26.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:46.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:48.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:51:50.439] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:13.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:17.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:19.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:42.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:44.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:52:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:14.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:16.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:26.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:46.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:48.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:53:50.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:15.895] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:17.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:19.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:40.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:44.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:54:46.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:14.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:16.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:27.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:45.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:49.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:55:51.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:16.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:18.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:20.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:41.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:56:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:12.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:16.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:26.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:46.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:48.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:57:50.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:13.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:17.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:19.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:40.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:44.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:58:46.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:12.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:16.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:26.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:45.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:47.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T20:59:49.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:13.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:15.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:17.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:39.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:43.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:00:45.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:13.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:15.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:25.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:43.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:47.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:01:49.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:11.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:15.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:17.464] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:48.596] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:50.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:02:52.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:03:14.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:03:36.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:03:38.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:14.629] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:16.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:26.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:46.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:48.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:04:50.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:13.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:17.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:19.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:42.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:05:46.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:15.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:17.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:27.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:45.192] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:49.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:06:51.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:16.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:18.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:20.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:39.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:44.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:07:46.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:12.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:16.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:26.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:46.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:48.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:08:50.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:15.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:17.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:19.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:42.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:44.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:09:46.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:12.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:16.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:26.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:44.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:48.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:10:50.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:13.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:17.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:20.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:11:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:12:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:13.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:40.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:43.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:13:45.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:14.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:24.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:26.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:44.741] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:48.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:14:50.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:15.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:17.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:19.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:40.645] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:44.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:15:46.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:14.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:16.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:26.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:47.120] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:16:51.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:16.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:18.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:20.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:41.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:43.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:17:45.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:14.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:19.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:26.238] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:44.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:48.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:18:50.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:13.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:17.550] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:19.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:42.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:44.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:19:46.323] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:14.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:16.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:26.615] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:46.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:48.805] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:20:50.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:15.900] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:17.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:19.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:42.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:44.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:21:46.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:12.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:16.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:26.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:47.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:49.184] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:22:51.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:14.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:18.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:20.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:41.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:43.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:23:45.987] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:12.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:16.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:26.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:46.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:48.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:24:50.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:15.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:17.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:19.605] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:40.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:44.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:25:46.372] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:14.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:16.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:26.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:45.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:47.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:26:49.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:11.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:15.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:17.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:41.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:43.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:27:45.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:12.117] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:16.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:26.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:48.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:28:50.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:13.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:17.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:19.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:42.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:44.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:29:46.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:12.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:16.509] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:26.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:46.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:48.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:30:50.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:15.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:17.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:19.851] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:40.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:44.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:31:46.615] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:12.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:16.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:26.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:45.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:49.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:32:51.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:16.188] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:18.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:20.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:41.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:43.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:33:45.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:12.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:16.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:26.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:44.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:48.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:34:50.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:15.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:17.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:19.514] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:40.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:44.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:35:46.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:14.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:16.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:26.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:46.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:48.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:36:50.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:13.851] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:17.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:19.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:40.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:44.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:37:46.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:12.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:16.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:26.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:45.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:48.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:38:50.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:13.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:17.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:19.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:42.027] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:44.035] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:39:46.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:12.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:16.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:26.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:44.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:48.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:40:50.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:13.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:17.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:19.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:42.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:44.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:41:46.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:12.689] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:16.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:26.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:46.883] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:48.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:42:50.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:16.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:18.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:20.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:43:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:44.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:44:50.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:15.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:17.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:19.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:42.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:44.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:45:46.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:12.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:16.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:26.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:46.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:48.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:46:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:15.748] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:17.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:19.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:42.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:44.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:47:46.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:12.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:16.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:26.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:47.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:49.041] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:48:51.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:14.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:18.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:20.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:41.816] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:43.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:49:45.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:14.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:19.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:26.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:46.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:48.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:50:50.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:14.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:16.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:18.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:42.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:44.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:51:46.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:12.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:16.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:26.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:46.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:52:50.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:15.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:19.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:42.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:44.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:53:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:12.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:16.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:26.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:47.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:49.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:54:51.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:16.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:18.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:20.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:39.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:43.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:55:45.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:12.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:16.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:26.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:44.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:48.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:56:50.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:15.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:17.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:19.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:40.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:44.337] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:57:46.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:12.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:16.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:26.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:45.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:47.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:58:49.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:11.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:15.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:17.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:47.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:51.104] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T21:59:53.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:18.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:36.306] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:38.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:40.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:44.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:00:46.379] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:12.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:16.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:26.671] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:44.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:48.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:01:50.874] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:15.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:17.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:19.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:02:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:12.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:16.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:26.092] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:03:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:13.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:17.403] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:41.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:04:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:05:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:13.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:17.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:40.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:06:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:14.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:16.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:26.954] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:47.116] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:49.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:07:51.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:16.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:18.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:20.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:41.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:43.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:08:45.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:26.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:46.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:48.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:09:50.442] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:13.513] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:17.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:19.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:40.307] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:44.317] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:10:46.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:14.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:16.602] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:26.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:46.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:48.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:11:50.822] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:12.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:16.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:18.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:12:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:13.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:17.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:24.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:44.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:46.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:13:48.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:15.394] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:17.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:46.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:50.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:14:52.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:23.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:35.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:37.781] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:41.812] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:43.820] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:15:45.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:12.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:16.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:26.124] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:46.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:48.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:16:50.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:15.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:17.422] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:19.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:40.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:44.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:17:46.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:14.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:16.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:26.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:44.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:48.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:18:50.708] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:15.795] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:17.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:19.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:40.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:44.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:19:46.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:14.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:21.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:26.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:47.047] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:49.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:20:51.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:16.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:18.205] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:20.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:39.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:43.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:21:45.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:12.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:16.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:26.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:44.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:48.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:22:50.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:13.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:17.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:19.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:40.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:44.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:23:46.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:14.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:16.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:26.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:48.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:24:50.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:12.827] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:16.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:18.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:25:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:26:50.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:40.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:43.173] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:27:45.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:12.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:16.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:26.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:46.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:48.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:28:50.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:13.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:17.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:19.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:40.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:44.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:29:46.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:14.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:16.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:26.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:47.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:49.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:30:51.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:13.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:17.264] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:19.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:42.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:44.034] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:31:46.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:12.308] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:16.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:26.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:46.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:48.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:32:50.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:13.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:17.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:19.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:40.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:44.418] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:33:46.426] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:12.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:16.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:26.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:44.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:48.907] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:34:50.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:15.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:18.032] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:20.042] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:35:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:44.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:36:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:14.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:16.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:18.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:42.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:44.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:37:46.268] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:14.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:16.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:26.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:48.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:38:50.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:15.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:17.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:19.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:42.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:44.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:39:46.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:12.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:16.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:26.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:40:51.146] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:15.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:17.245] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:19.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:42.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:44.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:41:46.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:14.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:16.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:26.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:46.480] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:48.508] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:42:50.523] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:13.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:17.626] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:19.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:42.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:44.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:43:46.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:12.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:16.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:26.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:44.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:48.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:44:50.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:13.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:17.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:20.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:45:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:12.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:15.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:25.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:45.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:47.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:46:49.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:11.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:15.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:17.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:48.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:50.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:47:52.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:48:19.709] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:48:37.746] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:48:39.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:14.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:16.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:26.476] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:44.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:48.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:49:50.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:15.747] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:17.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:19.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:40.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:44.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:50:46.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:14.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:16.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:26.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:46.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:48.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:51:50.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:13.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:17.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:19.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:41.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:43.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:52:45.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:14.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:20.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:25.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:43.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:47.417] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:53:49.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:11.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:15.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:17.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:46.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:50.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:54:52.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:32.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:34.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:36.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:42.043] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:44.051] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:55:46.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:14.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:16.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:26.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:44.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:48.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:56:50.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:13.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:17.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:19.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:40.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:44.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:57:46.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:12.712] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:16.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:26.739] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:44.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:48.925] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:58:50.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:14.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:18.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:20.056] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T22:59:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:44.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:00:50.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:13.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:40.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:01:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:14.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:16.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:02:50.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:15.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:03:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:12.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:26.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:45.100] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:49.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:04:51.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:14.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:18.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:20.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:41.921] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:43.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:05:45.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:12.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:16.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:26.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:46.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:48.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:06:50.440] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:15.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:17.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:19.554] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:42.302] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:44.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:07:46.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:12.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:16.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:26.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:46.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:48.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:08:50.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:15.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:17.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:40.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:44.687] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:09:46.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:14.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:16.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:26.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:47.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:49.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:10:51.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:16.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:18.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:20.313] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:41.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:43.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:11:45.976] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:12.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:16.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:26.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:44.425] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:48.461] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:12:50.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:13.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:17.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:19.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:40.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:44.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:13:46.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:12.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:16.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:26.654] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:46.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:48.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:14:50.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:15.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:17.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:19.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:42.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:44.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:15:46.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:13.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:17.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:27.031] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:45.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:49.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:16:51.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:14.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:18.328] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:20.352] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:41.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:44.007] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:17:46.016] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:14.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:16.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:26.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:44.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:48.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:18:50.513] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:13.584] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:17.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:19.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:40.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:44.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:19:46.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:14.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:16.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:45.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:47.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:20:49.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:11.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:15.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:18.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:41.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:43.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:21:45.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:12.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:16.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:26.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:44.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:48.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:22:50.368] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:14.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:16.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:18.489] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:40.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:44.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:23:46.353] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:14.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:16.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:26.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:44.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:48.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:24:50.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:15.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:17.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:19.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:40.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:44.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:25:46.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:12.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:17.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:27.028] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:47.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:49.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:26:51.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:14.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:18.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:20.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:41.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:44.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:27:46.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:12.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:16.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:26.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:46.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:48.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:28:50.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:15.590] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:17.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:19.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:40.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:44.383] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:29:46.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:12.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:16.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:26.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:44.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:48.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:30:50.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:15.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:17.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:20.001] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:31:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:32:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:13.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:17.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:19.405] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:44.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:33:46.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:14.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:16.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:26.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:46.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:48.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:34:50.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:15.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:17.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:19.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:42.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:44.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:35:46.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:14.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:19.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:26.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:45.010] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:49.040] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:36:51.053] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:14.125] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:18.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:20.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:41.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:43.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:37:45.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:12.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:16.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:26.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:46.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:48.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:38:50.328] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:15.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:17.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:19.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:42.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:44.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:39:46.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:12.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:19.497] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:26.512] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:46.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:48.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:40:50.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:13.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:17.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:19.834] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:40.583] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:44.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:41:46.601] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:14.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:16.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:26.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:47.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:49.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:42:51.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:14.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:18.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:20.222] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:41.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:43.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:43:45.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:13.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:15.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:25.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:45.345] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:47.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:44:49.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:11.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:15.478] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:17.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:48.638] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:50.653] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:45:52.682] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:46:17.811] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:46:36.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:46:38.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:12.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:16.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:26.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:44.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:48.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:47:50.896] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:15.972] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:18.008] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:20.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:48:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:26.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:46.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:48.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:49:50.296] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:17.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:41.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:43.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:50:45.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:48.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:51:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:17.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:19.890] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:52:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:14.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:16.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:26.938] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:47.090] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:49.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:53:51.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:18.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:20.257] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:39.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:43.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:54:45.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:14.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:16.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:26.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:44.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:48.397] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:55:50.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:15.492] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:17.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:19.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:42.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:44.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:56:46.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:14.561] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:19.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:26.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:46.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:48.779] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:57:50.793] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:15.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:17.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:19.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:42.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:44.667] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:58:46.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:12.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:16.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:26.971] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:45.123] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:49.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-27T23:59:51.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:14.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:18.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:39.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:43.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:00:45.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:12.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:16.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:26.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:46.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:48.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:01:50.456] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:15.529] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:17.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:19.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:42.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:44.331] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:02:46.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:14.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:19.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:26.635] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:44.786] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:48.823] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:03:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:13.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:17.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:19.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:40.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:44.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:04:46.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:14.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:20.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:27.019] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:47.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:49.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:05:51.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:16.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:18.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:20.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:39.984] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:43.994] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:06:46.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:14.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:16.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:26.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:46.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:48.486] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:07:50.500] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:13.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:17.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:19.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:42.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:44.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:08:46.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:12.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:16.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:26.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:46.838] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:48.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:09:50.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:13.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:17.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:19.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:10:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:13.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:15.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:25.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:45.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:47.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:11:49.292] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:11.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:15.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:17.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:46.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:50.582] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:12:52.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:13:34.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:13:36.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:13:38.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:12.574] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:16.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:26.604] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:46.765] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:48.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:14:50.807] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:13.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:17.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:19.924] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:42.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:44.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:15:46.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:12.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:16.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:26.981] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:47.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:49.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:16:51.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:16.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:18.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:20.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:41.947] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:43.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:17:45.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:12.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:16.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:26.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:44.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:48.447] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:18:50.460] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:13.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:17.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:19.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:40.329] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:44.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:19:46.347] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:12.613] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:16.625] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:26.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:46.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:48.825] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:20:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:15.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:17.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:19.958] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:40.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:44.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:21:46.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:14.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:16.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:27.014] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:49.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:22:51.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:14.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:18.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:20.338] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:39.982] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:43.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:23:46.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:12.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:16.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:26.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:45.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:47.484] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:24:49.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:11.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:15.595] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:17.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:48.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:25:52.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:26:19.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:26:37.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:26:41.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:14.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:16.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:26.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:44.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:48.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:27:50.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:15.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:18.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:20.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:28:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:29:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:13.360] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:40.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:44.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:30:46.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:12.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:16.453] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:26.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:46.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:48.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:31:50.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:15.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:17.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:19.792] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:40.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:44.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:32:46.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:12.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:16.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:26.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:45.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:49.039] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:33:51.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:14.120] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:18.151] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:20.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:41.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:43.829] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:34:45.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:12.102] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:16.113] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:26.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:44.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:48.319] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:35:50.333] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:13.406] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:17.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:19.452] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:42.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:44.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:36:46.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:12.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:16.494] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:26.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:46.669] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:48.699] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:37:50.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:15.785] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:17.809] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:19.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:40.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:44.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:38:46.593] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:14.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:19.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:26.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:47.045] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:49.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:39:51.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:14.160] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:18.186] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:20.209] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:39.855] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:43.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:40:45.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:12.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:15.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:25.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:44.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:46.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:41:48.370] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:11.443] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:15.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:17.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:47.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:49.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:42:51.675] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:19.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:33.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:35.860] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:40.078] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:44.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:43:46.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:12.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:16.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:26.392] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:46.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:48.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:44:50.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:13.668] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:17.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:19.717] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:42.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:44.474] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:45:46.482] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:14.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:19.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:26.776] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:44.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:48.964] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:46:50.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:16.052] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:18.075] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:20.096] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:41.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:43.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:47:45.760] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:48.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:48:50.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:14.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:16.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:18.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:40.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:44.272] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:49:46.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:12.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:16.558] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:26.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:48.761] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:50:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:13.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:17.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:42.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:44.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:51:46.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:12.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:16.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:47.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:49.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:52:51.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:14.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:20.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:41.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:43.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:53:45.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:12.201] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:16.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:26.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:46.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:48.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:54:50.430] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:13.499] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:17.524] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:19.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:40.300] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:44.310] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:55:46.318] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:14.586] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:19.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:26.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:44.771] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:48.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:56:50.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:15.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:17.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:19.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:42.680] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:44.688] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:57:46.696] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:12.963] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:16.974] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:26.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:45.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:48.179] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:58:50.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:15.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:17.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:19.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:40.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:44.068] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T00:59:46.076] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:12.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:16.354] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:26.371] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:46.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:48.559] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:00:50.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:13.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:17.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:19.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:40.441] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:44.451] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:01:46.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:14.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:16.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:26.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:44.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:48.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:02:50.957] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:14.030] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:18.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:20.076] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:41.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:43.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:03:45.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:46.250] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:04:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:15.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:17.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:42.156] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:44.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:05:46.172] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:12.438] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:16.450] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:26.466] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:45.618] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:47.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:06:49.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:11.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:15.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:17.785] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:46.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:50.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:07:52.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:31.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:34.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:36.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:40.378] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:08:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:12.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:16.673] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:44.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:48.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:09:50.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:15.975] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:18.003] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:20.013] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:10:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:44.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:11:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:13.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:19.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:40.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:12:45.170] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:12.537] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:26.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:44.725] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:48.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:13:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:15.840] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:17.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:42.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:44.641] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:14:46.649] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:12.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:26.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:45.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:49.130] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:15:51.143] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:15.212] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:17.235] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:19.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:40.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:44.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:16:46.020] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:14.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:16.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:26.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:46.472] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:48.501] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:17:50.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:15.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:17.612] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:19.633] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:40.378] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:44.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:18:46.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:14.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:19.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:26.694] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:44.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:48.883] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:19:50.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:15.969] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:17.993] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:20.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:20:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:46.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:48.279] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:21:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:15.366] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:17.395] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:22:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:12.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:46.732] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:23:50.774] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:15.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:19.893] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:40.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:44.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:24:46.662] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:12.929] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:16.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:26.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:47.122] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:49.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:25:51.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:14.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:18.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:20.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:41.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:43.940] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:26:45.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:14.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:16.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:26.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:45.399] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:47.435] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:27:49.448] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:11.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:15.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:17.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:46.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:50.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:28:52.749] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:29:19.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:29:36.915] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:29:38.937] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:14.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:16.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:26.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:44.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:48.946] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:30:50.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:16.033] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:18.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:20.080] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:41.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:43.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:31:45.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:14.061] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:16.070] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:44.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:32:50.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:15.362] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:17.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:19.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:40.152] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:43.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:33:45.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:14.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:16.548] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:44.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:34:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:15.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:17.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:19.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:40.632] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:35:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:16.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:45.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:36:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:16.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:18.261] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:20.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:41.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:43.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:37:45.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:12.202] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:16.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:26.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:44.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:47.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:38:49.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:11.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:15.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:17.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:48.685] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:50.695] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:39:52.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:33.875] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:35.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:37.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:39.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:43.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:40:45.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:12.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:25.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:45.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:47.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:41:49.467] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:13.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:15.576] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:17.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:46.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:50.740] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:42:52.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:43:17.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:43:36.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:43:38.956] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:14.751] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:19.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:26.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:44.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:48.967] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:44:50.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:14.050] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:18.077] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:20.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:41.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:43.758] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:45:45.766] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:14.064] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:16.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:46.260] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:48.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:46:50.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:14.374] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:16.407] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:18.419] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:40.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:44.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:47:46.284] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:12.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:16.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:26.581] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:46.743] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:48.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:48:50.784] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:13.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:17.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:19.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:42.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:44.658] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:49:46.666] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:14.934] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:16.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:26.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:45.127] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:49.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:50:51.169] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:14.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:18.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:20.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:39.932] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:43.942] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:51:45.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:14.219] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:16.230] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:26.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:46.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:48.432] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:52:50.446] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:13.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:17.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:19.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:42.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:44.322] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:53:46.330] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:12.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:16.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:26.627] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:44.780] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:48.817] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:54:50.830] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:12.901] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:18.949] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:55:45.735] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:14.063] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:16.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:26.089] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:48.280] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:56:50.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:13.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:17.389] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:19.413] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:40.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:43.168] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:57:45.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:12.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:16.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:26.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:46.737] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:48.763] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:58:50.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:15.848] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:19.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:42.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:44.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T01:59:46.660] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:14.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:16.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:26.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:45.114] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:49.142] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:00:51.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:16.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:18.265] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:20.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:01:45.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:12.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:15.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:25.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:43.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:47.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:02:49.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:13.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:15.521] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:17.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:48.677] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:50.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:03:52.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:04:14.846] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:04:37.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:04:39.914] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:13.589] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:15.598] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:25.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:43.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:47.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:05:49.816] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:14.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:16.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:18.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:43.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:06:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:26.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:48.278] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:07:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:13.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:17.391] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:19.412] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:40.157] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:43.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:08:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:12.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:16.551] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:44.729] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:48.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:09:50.773] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:15.845] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:19.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:42.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:44.647] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:10:46.655] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:14.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:19.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:26.951] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:45.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:11:51.153] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:16.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:20.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:41.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:43.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:12:45.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:14.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:16.216] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:26.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:46.388] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:48.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:13:50.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:15.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:17.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:19.556] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:42.304] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:44.312] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:14:46.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:14.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:16.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:26.614] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:46.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:48.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:15:50.818] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:15.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:17.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:19.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:40.683] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:44.693] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:16:46.701] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:14.968] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:19.980] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:26.995] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:45.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:49.182] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:17:51.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:16.269] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:18.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:20.314] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:41.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:43.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:18:45.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:12.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:16.256] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:26.273] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:46.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:48.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:19:50.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:13.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:17.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:19.587] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:42.334] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:44.342] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:20:46.350] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:12.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:16.628] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:26.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:44.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:48.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:21:50.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:13.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:17.941] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:19.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:42.713] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:44.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:22:46.731] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:14.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:21.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:26.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:46.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:48.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:23:50.229] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:13.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:17.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:19.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:41.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:43.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:24:45.111] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:12.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:16.488] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:26.504] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:44.661] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:48.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:25:50.706] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:13.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:17.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:19.824] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:42.572] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:44.580] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:26:46.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:12.852] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:16.863] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:26.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:47.046] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:49.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:27:51.086] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:16.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:18.194] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:20.203] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:39.847] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:43.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:28:45.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:12.129] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:16.141] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:26.158] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:46.309] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:48.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:29:50.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:13.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:17.454] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:19.477] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:42.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:44.233] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:30:46.241] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:14.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:16.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:26.536] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:46.698] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:48.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:31:50.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:13.806] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:17.833] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:19.856] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:42.603] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:44.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:32:46.619] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:12.884] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:16.897] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:26.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:47.072] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:49.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:33:51.115] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:14.183] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:18.211] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:20.234] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:41.881] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:43.889] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:34:45.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:12.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:16.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:26.191] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:46.355] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:48.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:35:50.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:13.465] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:17.496] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:19.511] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:42.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:44.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:36:46.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:12.541] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:16.553] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:44.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:37:50.769] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:12.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:16.861] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:18.885] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:38:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:14.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:46.249] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:48.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:39:50.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:13.357] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:17.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:19.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:41.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:43.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:40:45.171] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:12.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:16.549] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:26.567] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:46.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:48.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:41:50.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:15.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:17.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:44.643] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:42:46.651] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:14.918] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:19.930] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:26.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:45.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:49.131] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:43:51.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:14.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:44:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:14.195] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:16.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:26.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:44.378] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:48.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:45:50.423] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:13.493] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:17.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:19.540] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:42.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:44.291] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:46:46.299] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:12.568] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:16.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:26.597] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:46.753] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:48.788] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:47:50.801] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:15.873] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:19.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:42.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:44.672] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:48:46.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:14.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:19.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:26.977] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:45.136] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:49.164] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:49:51.177] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:16.258] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:18.288] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:41.944] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:43.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:50:45.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:12.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:16.237] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:26.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:44.415] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:48.444] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:51:50.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:15.535] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:17.565] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:19.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:42.327] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:52:46.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:12.611] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:16.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:26.639] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:44.800] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:48.828] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:53:50.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:15.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:17.936] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:19.959] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:42.714] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:44.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:54:46.730] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:12.998] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:17.009] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:27.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:47.190] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:49.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:55:51.231] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:16.315] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:18.340] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:20.349] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:41.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:44.004] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:56:46.012] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:12.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:16.285] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:26.301] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:46.457] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:48.490] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:57:50.503] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:15.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:17.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:19.622] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:40.367] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:44.377] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:58:46.385] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:12.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:16.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:26.681] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:44.837] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:47.872] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T02:59:49.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:13.965] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:15.993] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:18.002] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:41.851] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:43.859] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:00:45.867] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:12.134] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:16.145] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:26.163] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:46.324] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:48.351] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:01:50.365] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:13.436] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:17.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:19.485] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:42.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:44.240] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:02:46.248] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:14.518] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:16.527] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:26.545] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:44.704] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:48.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:03:50.750] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:12.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:16.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:18.864] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:40.710] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:44.720] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:04:46.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:14.996] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:17.005] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:27.022] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:46.176] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:48.210] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:05:50.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:13.294] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:17.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:19.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:42.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:44.099] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:06:46.107] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:12.373] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:16.384] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:26.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:44.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:48.594] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:07:50.607] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:13.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:17.703] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:19.724] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:40.469] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:44.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:08:46.487] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:14.756] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:18.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:25.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:45.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:47.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:09:49.985] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:13.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:17.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:19.101] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:41.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:43.858] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:10:45.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:12.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:16.144] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:26.161] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:44.320] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:48.348] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:11:50.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:13.429] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:17.456] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:19.479] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:42.227] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:44.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:12:46.244] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:12.510] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:16.522] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:26.539] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:46.702] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:48.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:13:50.742] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:12.810] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:16.842] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:18.859] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:40.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:14:46.723] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:14.990] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:17.000] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:27.017] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:47.181] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:49.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:15:51.223] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:14.293] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:17.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:19.339] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:41.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:43.095] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:16:45.103] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:12.470] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:16.481] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:26.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:44.663] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:48.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:17:50.705] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:13.775] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:17.799] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:19.821] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:42.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:44.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:18:46.585] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:14.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:19.866] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:26.882] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:47.036] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:49.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:19:51.088] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:16.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:18.199] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:20.208] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:41.854] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:43.862] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:20:45.870] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:13.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:15.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:25.166] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:43.321] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:46.356] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:21:48.369] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:13.459] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:15.483] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:17.491] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:46.631] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:50.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:22:52.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:23:19.813] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:23:36.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:23:38.878] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:12.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:16.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:26.691] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:44.857] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:48.886] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:24:50.899] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:12.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:16.992] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:19.015] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:41.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:43.770] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:25:45.778] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:12.062] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:16.074] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:26.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:48.277] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:26:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:15.364] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:17.387] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:19.410] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:40.155] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:43.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:27:45.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:14.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:16.552] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:26.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:44.733] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:48.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:28:50.777] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:13.844] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:17.869] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:19.892] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:42.640] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:44.648] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:29:46.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:12.922] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:16.933] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:26.950] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:47.106] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:49.135] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:30:51.149] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:16.224] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:18.252] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:20.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:39.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:43.923] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:31:45.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:14.198] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:24.215] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:26.226] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:46.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:48.414] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:32:50.428] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:13.498] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:19.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:42.295] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:44.303] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:33:46.311] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:12.577] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:16.588] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:26.606] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:46.762] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:48.790] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:34:50.803] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:13.871] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:17.903] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:19.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:40.665] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:44.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:35:46.684] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:14.952] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:16.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:26.978] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:45.128] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:48.162] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:36:50.175] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:13.253] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:17.283] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:19.298] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:42.048] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:44.057] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:37:46.065] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:14.332] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:16.341] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:26.359] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:44.515] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:48.546] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:38:50.560] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:12.630] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:16.656] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:18.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:42.526] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:44.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:39:46.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:12.808] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:16.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:26.836] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:46.999] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:49.025] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:40:51.038] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:15.110] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:17.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:19.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:41.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:43.911] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:41:45.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:12.185] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:16.197] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:26.213] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:46.375] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:48.402] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:42:50.416] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:13.485] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:17.519] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:19.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:42.281] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:44.289] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:43:46.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:14.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:16.573] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:26.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:46.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:48.782] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:44:50.797] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:15.868] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:17.898] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:19.913] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:40.659] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:44.670] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:45:46.678] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:14.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:16.955] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:26.973] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:45.137] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:49.165] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:46:51.178] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:16.251] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:18.282] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:20.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:41.945] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:43.953] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:47:45.962] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:12.228] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:16.239] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:26.255] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:44.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:48.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:48:50.458] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:13.532] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:17.564] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:19.578] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:42.325] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:44.335] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:49:46.343] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:12.609] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:16.620] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:26.636] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:46.798] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:48.826] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:50:50.839] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:12.910] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:16.935] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:18.960] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:41.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:43.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:51:45.736] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:12.055] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:16.066] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:26.084] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:44.242] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:48.274] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:52:50.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:15.358] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:17.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:19.404] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:40.150] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:43.159] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:53:45.167] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:14.538] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:16.547] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:26.566] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:44.715] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:48.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:54:50.767] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:15.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:17.879] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:19.888] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:40.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:44.644] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:55:46.652] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:14.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:19.931] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:26.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:47.112] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:49.139] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:56:51.154] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:16.232] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:18.262] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:20.271] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:39.909] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:43.919] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:57:45.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:12.193] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:16.204] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:26.220] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:46.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:48.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:58:50.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:15.502] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:17.533] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:19.542] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:40.287] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:44.297] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T03:59:46.305] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:14.575] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:18.592] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:29.617] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:43.707] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:45.727] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:00:47.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:17.891] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:21.917] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:23.939] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:42.382] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:44.390] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:01:46.398] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:12.664] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:16.676] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:26.692] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:46.850] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:48.880] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:02:50.894] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:15.966] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:17.989] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:20.011] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:41.718] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:43.726] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:03:45.734] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:12.060] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:16.071] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:26.087] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:46.247] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:48.276] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:04:50.290] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:14.363] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:16.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:18.409] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:42.259] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:44.267] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:05:46.275] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:14.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:19.555] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:26.569] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:44.719] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:48.755] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:06:50.768] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:15.841] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:17.865] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:19.887] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:42.634] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:44.642] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:07:46.650] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:12.916] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:16.927] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:26.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:45.105] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:49.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:08:51.147] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:16.217] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:18.243] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:20.266] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:41.912] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:43.920] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:09:45.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:12.189] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:16.200] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:26.218] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:44.380] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:48.408] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:10:50.421] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:15.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:19.544] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:23.563] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:42.721] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:46.738] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:11:50.754] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:12.420] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:16.437] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:23.455] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:42.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:46.579] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:12:50.599] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:16.722] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:31.752] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:33.791] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:39.832] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:43.849] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:13:54.877] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:11.926] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:15.961] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:21.986] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:43.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:45.073] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:14:49.091] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:16.187] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:25.207] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:38.236] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:42.254] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:46.270] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:15:50.286] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:12.376] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:20.396] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:28.411] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:48.506] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:52.534] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:16:56.570] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:16.657] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:20.700] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:24.716] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:17:47.783] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:04.819] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:06.853] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:15.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:23.928] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:27.943] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:41.979] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:45.997] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:18:50.018] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:16.174] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:25.225] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:29.246] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:19:53.316] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:01.346] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:03.386] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:14.424] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:22.445] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:26.463] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:42.520] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:46.557] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:20:48.571] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:21:23.759] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:21:40.804] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:21:44.835] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:14.543] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:18.562] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:30.591] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:42.679] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:46.712] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:22:50.728] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:15.843] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:32.876] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:36.902] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:40.948] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:44.970] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:48.991] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:23:53.546] error: *** JOB 14360421 ON gpua094 CANCELLED AT 2025-12-28T04:23:53 DUE to SIGNAL Terminated *** +[2025-12-28T04:24:01.026] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:24:17.059] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:24:21.097] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:24:26.133] error: NVML: Failed to get usage(999): Unknown Error +[2025-12-28T04:26:24.001] error: namespace_p_join: open failed for /local/slurmjobs/14360421/.ns: No such file or directory +[2025-12-28T04:26:24.001] error: namespace_g_join(14360421): No such file or directory +s-nail: Cannot find a usable character set to encode message: No such entry, file or directory +s-nail: ... message not sent +[2025-12-28T04:26:25.000] error: *** JOB 14360421 STEPD TERMINATED ON gpua094 AT 2025-12-28T04:26:23 DUE TO JOB NOT ENDING WITH SIGNALS *** +[2025-12-28T04:26:33.135] error: Container 2393516 in cgroup plugin has 2 processes, giving up after 127 sec diff --git a/collaborativeagents/slurm/logs/rag_vector_14360422.err b/collaborativeagents/slurm/logs/rag_vector_14360422.err new file mode 100644 index 0000000..8195f29 --- /dev/null +++ b/collaborativeagents/slurm/logs/rag_vector_14360422.err @@ -0,0 +1,129 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:59:04,500 - INFO - Loaded dataset: mmlu +2025-12-27 01:59:04,500 - INFO - Loaded dataset: aime +2025-12-27 01:59:04,500 - INFO - Loaded dataset: math-hard +2025-12-27 01:59:04,500 - INFO - Loaded dataset: humaneval +2025-12-27 01:59:04,516 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:59:04,517 - INFO - Running method: rag_vector +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.76s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.40s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.15s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.67s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.97s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.84s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.56s/it] +2025-12-27 01:59:47,035 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:08<01:04, 8.09s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:18<01:07, 9.62s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:29<01:00, 10.11s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:40<00:51, 10.29s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:56<00:49, 12.41s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:06<00:35, 11.72s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:15<00:21, 10.95s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:22<00:09, 9.70s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:25<00:00, 7.58s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:25<00:00, 9.55s/it] +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-27 02:05:38,040 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:26:36,031 - WARNING - User agent failed to respond at turn 2 +2025-12-27 02:49:12,707 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:49:16,867 - INFO - Profile 2/30 +2025-12-27 03:15:00,486 - WARNING - User agent failed to respond at turn 3 +2025-12-27 03:26:15,013 - INFO - Profile 3/30 +2025-12-27 04:03:36,467 - WARNING - User agent failed to respond at turn 2 +2025-12-27 04:08:15,791 - INFO - Profile 4/30 +2025-12-27 04:22:36,794 - WARNING - User agent failed to respond at turn 2 +2025-12-27 04:25:15,739 - WARNING - User agent failed to respond at turn 3 +2025-12-27 05:08:15,379 - INFO - Profile 5/30 +2025-12-27 05:45:53,211 - INFO - Profile 6/30 +2025-12-27 06:37:29,994 - INFO - Profile 7/30 +2025-12-27 06:44:34,982 - WARNING - User agent failed to respond at turn 5 +2025-12-27 07:07:39,808 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:24:40,817 - WARNING - User agent failed to respond at turn 2 +2025-12-27 07:40:10,779 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:40:10,781 - INFO - Profile 8/30 +2025-12-27 07:42:53,487 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:59:51,049 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:20:46,161 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:37:34,229 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:37:34,231 - INFO - Profile 9/30 +2025-12-27 09:21:28,395 - INFO - Profile 10/30 +2025-12-27 09:47:44,540 - WARNING - User agent failed to respond at turn 4 +2025-12-27 10:17:23,198 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:24:45,981 - WARNING - User agent failed to respond at turn 4 +2025-12-27 10:41:24,671 - INFO - Profile 11/30 +2025-12-27 11:38:47,631 - INFO - Profile 12/30 +2025-12-27 12:19:31,005 - WARNING - User agent failed to respond at turn 4 +2025-12-27 12:31:03,630 - INFO - Profile 13/30 +2025-12-27 12:54:50,025 - WARNING - User agent failed to respond at turn 6 +2025-12-27 12:58:36,759 - WARNING - User agent failed to respond at turn 3 +2025-12-27 13:02:04,376 - WARNING - User agent failed to respond at turn 5 +2025-12-27 13:14:20,302 - INFO - Profile 14/30 +2025-12-27 13:31:29,268 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:34:19,996 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:02:04,997 - INFO - Profile 15/30 +2025-12-27 14:21:27,253 - WARNING - User agent failed to respond at turn 4 +2025-12-27 14:41:02,004 - INFO - Profile 16/30 +2025-12-27 15:11:47,767 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:15:59,218 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:23:18,670 - INFO - Profile 17/30 +2025-12-27 15:26:29,631 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:41:28,028 - WARNING - User agent failed to respond at turn 5 +2025-12-27 16:16:11,075 - WARNING - User agent failed to respond at turn 2 +2025-12-27 16:25:49,209 - INFO - Profile 18/30 +2025-12-27 17:14:30,971 - INFO - Profile 19/30 +2025-12-27 17:23:08,123 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:20:12,384 - INFO - Profile 20/30 +2025-12-27 18:24:15,177 - WARNING - User agent failed to respond at turn 3 +2025-12-27 18:31:50,579 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:39:24,278 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:51:42,364 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:59:25,266 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:14:22,797 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:34:17,612 - INFO - Profile 21/30 +2025-12-27 19:56:12,348 - WARNING - User agent failed to respond at turn 2 +2025-12-27 20:03:59,021 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:10:11,631 - WARNING - User agent failed to respond at turn 6 +2025-12-27 20:26:56,963 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:30:31,496 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:34:38,513 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:53:09,810 - INFO - Profile 22/30 +2025-12-27 21:48:05,766 - INFO - Profile 23/30 +2025-12-27 22:28:29,304 - WARNING - User agent failed to respond at turn 4 +2025-12-27 22:59:13,258 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:59:13,261 - INFO - Profile 24/30 +2025-12-27 23:06:39,301 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:19:24,621 - WARNING - User agent failed to respond at turn 2 +2025-12-27 23:38:18,263 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:55:20,391 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:55:20,393 - INFO - Profile 25/30 +2025-12-28 00:02:06,164 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:08:19,823 - WARNING - User agent failed to respond at turn 2 +2025-12-28 00:23:05,450 - WARNING - User agent failed to respond at turn 2 +2025-12-28 00:52:08,278 - INFO - Profile 26/30 +2025-12-28 00:55:08,573 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:00:58,753 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:03:09,618 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:19:09,162 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:35:04,963 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:37:11,449 - WARNING - User agent failed to respond at turn 2 +2025-12-28 01:40:14,204 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:50:50,908 - INFO - Profile 27/30 +2025-12-28 01:53:17,203 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:11:07,167 - WARNING - User agent failed to respond at turn 4 +2025-12-28 02:24:30,855 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:48:24,373 - WARNING - User agent failed to respond at turn 4 +2025-12-28 02:48:24,376 - INFO - Profile 28/30 +2025-12-28 03:09:58,744 - WARNING - User agent failed to respond at turn 4 +2025-12-28 03:18:04,465 - WARNING - User agent failed to respond at turn 3 +2025-12-28 03:31:51,389 - WARNING - User agent failed to respond at turn 7 +2025-12-28 03:44:54,395 - INFO - Profile 29/30 +2025-12-28 04:17:56,139 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:51:52,370 - INFO - Profile 30/30 +2025-12-28 05:22:33,090 - WARNING - User agent failed to respond at turn 4 +2025-12-28 05:25:11,094 - WARNING - User agent failed to respond at turn 2 +2025-12-28 05:35:41,114 - INFO - Report saved to ../results/rag_vector_20251227_015842/20251227_015904/report.md diff --git a/collaborativeagents/slurm/logs/reflection_14360424.err b/collaborativeagents/slurm/logs/reflection_14360424.err new file mode 100644 index 0000000..551acf5 --- /dev/null +++ b/collaborativeagents/slurm/logs/reflection_14360424.err @@ -0,0 +1,101 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 02:02:07,264 - INFO - Loaded dataset: mmlu +2025-12-27 02:02:07,264 - INFO - Loaded dataset: aime +2025-12-27 02:02:07,264 - INFO - Loaded dataset: math-hard +2025-12-27 02:02:07,264 - INFO - Loaded dataset: humaneval +2025-12-27 02:02:07,278 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 02:02:07,278 - INFO - Running method: reflection +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.51s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.24s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.47s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.57s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +2025-12-27 02:02:32,181 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:04<00:33, 4.24s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:09<00:32, 4.71s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:14<00:29, 4.91s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:19<00:25, 5.00s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:25<00:21, 5.32s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:30<00:15, 5.23s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [00:35<00:10, 5.14s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [00:38<00:04, 4.62s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:40<00:00, 3.75s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:40<00:00, 4.54s/it] +2025-12-27 02:39:58,321 - WARNING - User agent failed to respond at turn 4 +2025-12-27 02:51:18,723 - INFO - Profile 2/30 +2025-12-27 03:42:18,671 - INFO - Profile 3/30 +2025-12-27 04:06:36,514 - WARNING - User agent failed to respond at turn 5 +2025-12-27 04:22:16,874 - WARNING - User agent failed to respond at turn 9 +2025-12-27 04:25:25,486 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:34:09,765 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:47:50,266 - INFO - Profile 4/30 +2025-12-27 05:08:18,870 - WARNING - User agent failed to respond at turn 2 +2025-12-27 06:04:25,163 - INFO - Profile 5/30 +2025-12-27 06:46:34,782 - INFO - Profile 6/30 +2025-12-27 07:37:47,157 - INFO - Profile 7/30 +2025-12-27 07:41:44,319 - WARNING - User agent failed to respond at turn 5 +2025-12-27 07:44:57,141 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:31:18,391 - WARNING - User agent failed to respond at turn 4 +2025-12-27 08:53:24,527 - INFO - Profile 8/30 +2025-12-27 09:02:17,811 - WARNING - User agent failed to respond at turn 2 +2025-12-27 09:10:00,697 - WARNING - User agent failed to respond at turn 2 +2025-12-27 10:06:01,427 - INFO - Profile 9/30 +2025-12-27 10:10:10,794 - WARNING - User agent failed to respond at turn 2 +2025-12-27 10:54:26,451 - INFO - Profile 10/30 +2025-12-27 11:45:46,800 - INFO - Profile 11/30 +2025-12-27 12:11:54,366 - WARNING - User agent failed to respond at turn 4 +2025-12-27 12:37:49,938 - WARNING - User agent failed to respond at turn 5 +2025-12-27 12:53:20,643 - INFO - Profile 12/30 +2025-12-27 13:21:19,712 - WARNING - User agent failed to respond at turn 3 +2025-12-27 13:52:22,200 - INFO - Profile 13/30 +2025-12-27 14:32:11,397 - WARNING - User agent failed to respond at turn 1 +2025-12-27 14:40:54,805 - INFO - Profile 14/30 +2025-12-27 15:32:12,165 - INFO - Profile 15/30 +2025-12-27 16:24:14,011 - INFO - Profile 16/30 +2025-12-27 17:10:39,805 - INFO - Profile 17/30 +2025-12-27 17:50:48,532 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:17:20,986 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:17:39,723 - INFO - Profile 18/30 +2025-12-27 18:18:10,896 - WARNING - User agent failed to respond at turn 0 +2025-12-27 18:50:45,817 - WARNING - User agent failed to respond at turn 2 +2025-12-27 19:18:30,976 - INFO - Profile 19/30 +2025-12-27 20:21:34,652 - INFO - Profile 20/30 +2025-12-27 20:26:57,365 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:30:35,756 - WARNING - User agent failed to respond at turn 5 +2025-12-27 20:42:35,308 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:48:30,267 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:51:42,151 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:55:16,916 - WARNING - User agent failed to respond at turn 4 +2025-12-27 21:17:50,813 - WARNING - User agent failed to respond at turn 4 +2025-12-27 21:26:06,307 - WARNING - User agent failed to respond at turn 7 +2025-12-27 21:50:43,671 - INFO - Profile 21/30 +2025-12-27 22:35:18,540 - WARNING - User agent failed to respond at turn 5 +2025-12-27 22:53:01,391 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:13:29,157 - INFO - Profile 22/30 +2025-12-27 23:51:06,223 - WARNING - User agent failed to respond at turn 8 +2025-12-28 00:17:23,488 - INFO - Profile 23/30 +2025-12-28 01:19:07,558 - WARNING - User agent failed to respond at turn 10 +2025-12-28 01:37:49,690 - WARNING - User agent failed to respond at turn 6 +2025-12-28 01:38:08,367 - INFO - Profile 24/30 +2025-12-28 02:41:38,285 - INFO - Profile 25/30 +2025-12-28 03:52:11,789 - INFO - Profile 26/30 +2025-12-28 04:23:39,557 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:34:09,940 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:46:57,468 - WARNING - User agent failed to respond at turn 3 +2025-12-28 05:05:43,353 - INFO - Profile 27/30 +2025-12-28 05:38:16,855 - WARNING - User agent failed to respond at turn 2 +2025-12-28 06:17:59,240 - INFO - Profile 28/30 +2025-12-28 06:23:46,429 - WARNING - User agent failed to respond at turn 3 +2025-12-28 06:40:27,991 - WARNING - User agent failed to respond at turn 4 +2025-12-28 07:06:02,435 - WARNING - User agent failed to respond at turn 3 +2025-12-28 07:16:42,790 - WARNING - User agent failed to respond at turn 4 +2025-12-28 07:34:31,553 - INFO - Profile 29/30 +2025-12-28 08:28:07,646 - WARNING - User agent failed to respond at turn 4 +2025-12-28 08:53:34,126 - INFO - Profile 30/30 +2025-12-28 09:32:37,813 - WARNING - User agent failed to respond at turn 7 +2025-12-28 09:40:54,513 - INFO - Report saved to ../results/reflection_20251227_020146/20251227_020207/report.md diff --git a/collaborativeagents/slurm/logs/reflection_grpo_14360425.err b/collaborativeagents/slurm/logs/reflection_grpo_14360425.err new file mode 100644 index 0000000..3b427bf --- /dev/null +++ b/collaborativeagents/slurm/logs/reflection_grpo_14360425.err @@ -0,0 +1,119 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 02:31:09,224 - INFO - Loaded dataset: mmlu +2025-12-27 02:31:09,224 - INFO - Loaded dataset: aime +2025-12-27 02:31:09,224 - INFO - Loaded dataset: math-hard +2025-12-27 02:31:09,224 - INFO - Loaded dataset: humaneval +2025-12-27 02:31:09,299 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 02:31:09,299 - INFO - Running method: reflection_grpo +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.49s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.14s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.13s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.07s/it] +2025-12-27 02:31:33,591 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:03<00:31, 3.95s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:08<00:31, 4.48s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:13<00:27, 4.63s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:18<00:23, 4.75s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:23<00:19, 4.93s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:29<00:15, 5.04s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [00:33<00:09, 4.96s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [00:37<00:04, 4.46s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 3.63s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [00:39<00:00, 4.34s/it] +2025-12-27 03:43:56,265 - WARNING - User agent failed to respond at turn 4 +2025-12-27 03:44:19,363 - INFO - Profile 2/30 +2025-12-27 04:41:18,679 - INFO - Profile 3/30 +2025-12-27 04:50:08,015 - WARNING - User agent failed to respond at turn 6 +2025-12-27 05:37:09,400 - WARNING - User agent failed to respond at turn 4 +2025-12-27 05:39:22,155 - INFO - Profile 4/30 +2025-12-27 05:51:40,082 - WARNING - User agent failed to respond at turn 4 +2025-12-27 06:30:54,910 - WARNING - User agent failed to respond at turn 4 +2025-12-27 06:55:12,778 - INFO - Profile 5/30 +2025-12-27 07:48:39,008 - INFO - Profile 6/30 +2025-12-27 08:01:42,219 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:23:42,492 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:54:04,212 - INFO - Profile 7/30 +2025-12-27 08:58:13,539 - WARNING - User agent failed to respond at turn 3 +2025-12-27 09:24:36,991 - WARNING - User agent failed to respond at turn 7 +2025-12-27 10:01:43,345 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:04:41,897 - WARNING - User agent failed to respond at turn 2 +2025-12-27 10:20:11,751 - INFO - Profile 8/30 +2025-12-27 11:08:02,876 - WARNING - User agent failed to respond at turn 4 +2025-12-27 11:20:28,004 - WARNING - User agent failed to respond at turn 5 +2025-12-27 11:46:14,996 - WARNING - User agent failed to respond at turn 4 +2025-12-27 11:46:33,648 - INFO - Profile 9/30 +2025-12-27 12:22:26,369 - WARNING - User agent failed to respond at turn 9 +2025-12-27 12:56:13,166 - INFO - Profile 10/30 +2025-12-27 13:02:01,791 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:24:51,498 - WARNING - User agent failed to respond at turn 3 +2025-12-27 14:16:50,083 - INFO - Profile 11/30 +2025-12-27 14:27:09,697 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:40:22,936 - INFO - Profile 12/30 +2025-12-27 15:52:57,164 - WARNING - User agent failed to respond at turn 5 +2025-12-27 16:28:17,345 - WARNING - User agent failed to respond at turn 4 +2025-12-27 16:50:21,596 - INFO - Profile 13/30 +2025-12-27 17:41:49,444 - WARNING - User agent failed to respond at turn 4 +2025-12-27 17:50:43,295 - INFO - Profile 14/30 +2025-12-27 18:08:38,210 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:44:39,617 - WARNING - User agent failed to respond at turn 8 +2025-12-27 18:47:39,503 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:00:23,116 - INFO - Profile 15/30 +2025-12-27 19:12:53,841 - WARNING - User agent failed to respond at turn 4 +2025-12-27 20:03:36,023 - INFO - Profile 16/30 +2025-12-27 20:50:11,725 - INFO - Profile 17/30 +2025-12-27 20:54:36,277 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:18:15,804 - INFO - Profile 18/30 +2025-12-27 22:40:24,135 - WARNING - User agent failed to respond at turn 3 +2025-12-27 23:04:20,252 - WARNING - User agent failed to respond at turn 4 +2025-12-27 23:23:13,204 - INFO - Profile 19/30 +2025-12-28 00:30:41,183 - INFO - Profile 20/30 +2025-12-28 01:13:04,372 - WARNING - User agent failed to respond at turn 6 +2025-12-28 01:21:59,883 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:47:59,918 - WARNING - User agent failed to respond at turn 7 +2025-12-28 01:53:24,077 - WARNING - User agent failed to respond at turn 5 +2025-12-28 02:13:56,170 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:14:14,770 - INFO - Profile 21/30 +2025-12-28 03:20:09,605 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:10:58,912 - INFO - Profile 22/30 +2025-12-28 05:16:04,670 - WARNING - User agent failed to respond at turn 4 +2025-12-28 05:29:32,044 - INFO - Profile 23/30 +2025-12-28 05:46:53,577 - WARNING - User agent failed to respond at turn 6 +2025-12-28 05:57:05,360 - WARNING - User agent failed to respond at turn 6 +2025-12-28 06:14:11,895 - WARNING - User agent failed to respond at turn 5 +2025-12-28 06:21:21,665 - WARNING - User agent failed to respond at turn 3 +2025-12-28 06:43:49,754 - INFO - Profile 24/30 +2025-12-28 06:56:35,737 - WARNING - User agent failed to respond at turn 3 +2025-12-28 07:54:52,613 - INFO - Profile 25/30 +2025-12-28 08:24:24,212 - WARNING - User agent failed to respond at turn 2 +2025-12-28 09:01:32,435 - INFO - Profile 26/30 +2025-12-28 09:24:20,607 - WARNING - User agent failed to respond at turn 4 +2025-12-28 09:28:45,402 - WARNING - User agent failed to respond at turn 3 +2025-12-28 09:31:07,307 - WARNING - User agent failed to respond at turn 2 +2025-12-28 09:37:47,214 - WARNING - User agent failed to respond at turn 2 +2025-12-28 09:49:55,833 - WARNING - User agent failed to respond at turn 3 +2025-12-28 09:54:03,278 - WARNING - User agent failed to respond at turn 2 +2025-12-28 10:13:11,944 - INFO - Profile 27/30 +2025-12-28 10:34:58,991 - WARNING - User agent failed to respond at turn 5 +2025-12-28 10:42:04,222 - WARNING - User agent failed to respond at turn 4 +2025-12-28 10:49:59,056 - WARNING - User agent failed to respond at turn 3 +2025-12-28 11:22:55,596 - WARNING - User agent failed to respond at turn 2 +2025-12-28 11:36:59,263 - INFO - Profile 28/30 +2025-12-28 11:49:22,297 - WARNING - User agent failed to respond at turn 6 +2025-12-28 11:52:54,358 - WARNING - User agent failed to respond at turn 3 +2025-12-28 12:04:04,093 - WARNING - User agent failed to respond at turn 4 +2025-12-28 12:11:08,251 - WARNING - User agent failed to respond at turn 3 +2025-12-28 12:17:19,369 - WARNING - User agent failed to respond at turn 6 +2025-12-28 12:20:37,784 - WARNING - User agent failed to respond at turn 4 +2025-12-28 12:33:13,220 - WARNING - User agent failed to respond at turn 4 +2025-12-28 12:41:36,621 - WARNING - User agent failed to respond at turn 5 +2025-12-28 12:44:34,822 - WARNING - User agent failed to respond at turn 3 +2025-12-28 12:50:56,698 - WARNING - User agent failed to respond at turn 6 +2025-12-28 12:56:12,269 - INFO - Profile 29/30 +2025-12-28 14:22:05,011 - INFO - Profile 30/30 +2025-12-28 15:13:10,367 - INFO - Report saved to ../results/reflection_grpo_20251227_023047/20251227_023109/report.md diff --git a/collaborativeagents/slurm/logs/run_collab_baselines_14355966.err b/collaborativeagents/slurm/logs/run_collab_baselines_14355966.err new file mode 100644 index 0000000..6d3191e --- /dev/null +++ b/collaborativeagents/slurm/logs/run_collab_baselines_14355966.err @@ -0,0 +1,14466 @@ +2025-12-25 10:37:47,003 - INFO - Loaded dataset: math-500 +2025-12-25 10:37:47,143 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 10:37:47,144 - INFO - Running method: contextual +2025-12-25 10:37:51,592 - INFO - Profile 1/20 +2025-12-25 10:37:53,048 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,062 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,063 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,066 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,067 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,069 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,070 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,073 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,074 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,076 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,077 - INFO - Profile 2/20 +2025-12-25 10:37:53,077 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,080 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,081 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,083 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,084 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,087 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,088 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,090 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,091 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,094 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,094 - INFO - Profile 3/20 +2025-12-25 10:37:53,094 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,097 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,098 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,101 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,101 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,104 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,105 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,107 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,108 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,111 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,111 - INFO - Profile 4/20 +2025-12-25 10:37:53,112 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,114 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,115 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,118 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,118 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,121 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,122 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,124 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,125 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,128 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,128 - INFO - Profile 5/20 +2025-12-25 10:37:53,129 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,131 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,132 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,135 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,136 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,138 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,139 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,142 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,142 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,145 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,145 - INFO - Profile 6/20 +2025-12-25 10:37:53,146 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,149 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,149 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,152 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,153 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,156 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,156 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,159 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,160 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,162 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,163 - INFO - Profile 7/20 +2025-12-25 10:37:53,163 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,166 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,167 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,169 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,170 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,173 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,174 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,176 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,177 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,180 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,180 - INFO - Profile 8/20 +2025-12-25 10:37:53,181 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,184 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,184 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,187 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,188 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,190 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,191 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,194 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,195 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,198 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,198 - INFO - Profile 9/20 +2025-12-25 10:37:53,198 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,201 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,202 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,205 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,206 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,208 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,209 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,212 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,213 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,215 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,215 - INFO - Profile 10/20 +2025-12-25 10:37:53,216 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,219 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,220 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,222 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,223 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,226 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,227 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,230 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,231 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,233 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,233 - INFO - Profile 11/20 +2025-12-25 10:37:53,234 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,237 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,238 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,240 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,241 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,244 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,245 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,247 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,248 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,251 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,251 - INFO - Profile 12/20 +2025-12-25 10:37:53,252 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,255 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,255 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,258 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,259 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,262 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,263 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,265 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,266 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,269 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,269 - INFO - Profile 13/20 +2025-12-25 10:37:53,270 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,273 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,273 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,276 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,277 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,280 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,281 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,283 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,284 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,287 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,287 - INFO - Profile 14/20 +2025-12-25 10:37:53,288 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,291 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,291 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,294 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,295 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,298 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,299 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,301 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,302 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,305 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,305 - INFO - Profile 15/20 +2025-12-25 10:37:53,306 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,309 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,310 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,312 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,313 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,316 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,317 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,320 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,321 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,323 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,323 - INFO - Profile 16/20 +2025-12-25 10:37:53,324 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,327 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,328 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,331 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,332 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,334 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,335 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,338 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,339 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,341 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,342 - INFO - Profile 17/20 +2025-12-25 10:37:53,343 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,345 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,346 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,349 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,350 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,352 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,354 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,356 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,357 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,360 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,360 - INFO - Profile 18/20 +2025-12-25 10:37:53,361 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,364 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,365 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,367 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,368 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,371 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,372 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,375 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,376 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,378 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,379 - INFO - Profile 19/20 +2025-12-25 10:37:53,380 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,382 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,383 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,386 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,387 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,390 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,391 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,393 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,394 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,397 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,397 - INFO - Profile 20/20 +2025-12-25 10:37:53,398 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,401 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,402 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,405 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,406 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,408 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,409 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,412 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,413 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,416 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/contextual_adapter.py", line 93, in generate_response + response = self._agent.generate_collaborator_response(context) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,422 - INFO - Running method: reflection +2025-12-25 10:37:53,427 - INFO - Profile 1/20 +2025-12-25 10:37:53,428 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,431 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,432 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,435 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,435 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,438 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,439 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,441 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,442 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,445 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,445 - INFO - Profile 2/20 +2025-12-25 10:37:53,445 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,448 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,449 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,451 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,452 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,455 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,455 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,458 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,459 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,461 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,462 - INFO - Profile 3/20 +2025-12-25 10:37:53,462 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,465 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,466 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,468 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,469 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,472 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,472 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,475 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,476 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,478 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,478 - INFO - Profile 4/20 +2025-12-25 10:37:53,479 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,482 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,482 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,485 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,486 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,488 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,489 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,492 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,493 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,495 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,495 - INFO - Profile 5/20 +2025-12-25 10:37:53,496 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,499 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,499 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,502 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,503 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,505 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,506 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,509 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,509 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,513 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,513 - INFO - Profile 6/20 +2025-12-25 10:37:53,514 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,517 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,517 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,520 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,521 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,523 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,524 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,527 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,527 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,530 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,530 - INFO - Profile 7/20 +2025-12-25 10:37:53,531 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,534 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,534 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,537 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,538 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,540 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,541 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,544 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,544 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,547 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,547 - INFO - Profile 8/20 +2025-12-25 10:37:53,548 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,550 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,551 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,554 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,554 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,557 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,558 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,560 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,561 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,564 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,564 - INFO - Profile 9/20 +2025-12-25 10:37:53,565 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,567 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,568 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,571 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,571 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,574 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,575 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,577 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,578 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,581 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,581 - INFO - Profile 10/20 +2025-12-25 10:37:53,581 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,584 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,585 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,587 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,588 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,591 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,591 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,594 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,595 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,597 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,598 - INFO - Profile 11/20 +2025-12-25 10:37:53,598 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,601 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,602 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,604 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,605 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,608 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,608 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,611 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,612 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,614 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,614 - INFO - Profile 12/20 +2025-12-25 10:37:53,615 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,618 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,618 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,621 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,622 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,624 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,625 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,628 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,628 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,631 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,631 - INFO - Profile 13/20 +2025-12-25 10:37:53,632 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,635 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,635 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,638 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,639 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,641 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,642 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,645 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,645 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,648 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,648 - INFO - Profile 14/20 +2025-12-25 10:37:53,649 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,651 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,652 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,655 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,655 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,658 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,659 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,661 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,662 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,665 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,665 - INFO - Profile 15/20 +2025-12-25 10:37:53,666 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,668 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,669 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,672 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,672 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,675 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,676 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,678 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,679 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,682 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,682 - INFO - Profile 16/20 +2025-12-25 10:37:53,682 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,685 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,686 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,688 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,689 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,692 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,692 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,695 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,696 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,698 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,699 - INFO - Profile 17/20 +2025-12-25 10:37:53,699 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,702 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,703 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,705 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,706 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,709 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,709 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,712 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,713 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,715 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,715 - INFO - Profile 18/20 +2025-12-25 10:37:53,716 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,719 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,719 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,722 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,723 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,725 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,726 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,729 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,729 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,732 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,732 - INFO - Profile 19/20 +2025-12-25 10:37:53,733 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,735 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,736 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,739 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,739 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,742 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,743 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,746 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,746 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,749 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,749 - INFO - Profile 20/20 +2025-12-25 10:37:53,750 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,752 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,753 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,756 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,756 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,759 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,760 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,762 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,763 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,766 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 93, in generate_collaborator_response + response = self.completion(messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,785 - INFO - Running method: reflection_grpo +2025-12-25 10:37:53,791 - INFO - Profile 1/20 +2025-12-25 10:37:53,791 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,795 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,795 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,798 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,799 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,801 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,802 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,805 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,805 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,808 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,808 - INFO - Profile 2/20 +2025-12-25 10:37:53,809 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,812 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,812 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,815 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,816 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,818 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,819 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,822 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,823 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,825 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,825 - INFO - Profile 3/20 +2025-12-25 10:37:53,826 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,829 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,829 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,832 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,833 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,835 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,836 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,839 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,840 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,842 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,842 - INFO - Profile 4/20 +2025-12-25 10:37:53,843 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,846 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,846 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,849 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,850 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,853 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,853 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,856 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,857 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,859 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,859 - INFO - Profile 5/20 +2025-12-25 10:37:53,860 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,863 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,864 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,866 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,867 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,870 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,870 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,873 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,874 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,876 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,876 - INFO - Profile 6/20 +2025-12-25 10:37:53,877 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,880 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,881 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,883 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,884 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,887 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,887 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,890 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,891 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,893 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,893 - INFO - Profile 7/20 +2025-12-25 10:37:53,894 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,897 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,897 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,900 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,901 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,903 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,904 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,907 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,908 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,910 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,910 - INFO - Profile 8/20 +2025-12-25 10:37:53,911 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,914 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,914 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,917 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,918 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,920 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,921 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,924 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,925 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,927 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,927 - INFO - Profile 9/20 +2025-12-25 10:37:53,928 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,931 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,931 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,934 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,935 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,937 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,938 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,941 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,942 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,944 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,944 - INFO - Profile 10/20 +2025-12-25 10:37:53,945 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,948 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,949 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,951 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,952 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,955 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,955 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,958 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,959 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,961 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,961 - INFO - Profile 11/20 +2025-12-25 10:37:53,962 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,965 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,966 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,968 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,969 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,972 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,972 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,975 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,976 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,978 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,979 - INFO - Profile 12/20 +2025-12-25 10:37:53,979 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,982 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,983 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,985 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,986 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,989 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,989 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,992 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,993 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,995 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:53,995 - INFO - Profile 13/20 +2025-12-25 10:37:53,996 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:53,999 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,000 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,002 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,003 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,006 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,006 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,009 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,010 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,012 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,012 - INFO - Profile 14/20 +2025-12-25 10:37:54,013 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,016 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,017 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,019 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,020 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,023 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,023 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,026 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,027 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,029 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,030 - INFO - Profile 15/20 +2025-12-25 10:37:54,030 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,033 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,034 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,036 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,037 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,040 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,040 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,043 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,044 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,047 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,047 - INFO - Profile 16/20 +2025-12-25 10:37:54,047 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,050 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,051 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,053 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,054 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,057 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,057 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,060 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,061 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,064 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,064 - INFO - Profile 17/20 +2025-12-25 10:37:54,064 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,067 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,068 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,070 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,071 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,074 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,074 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,077 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,078 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,080 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,081 - INFO - Profile 18/20 +2025-12-25 10:37:54,081 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,084 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,085 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,087 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,088 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,091 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,091 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,094 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,095 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,097 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,098 - INFO - Profile 19/20 +2025-12-25 10:37:54,098 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,101 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,102 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,104 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,105 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,108 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,108 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,111 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,112 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,114 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,115 - INFO - Profile 20/20 +2025-12-25 10:37:54,115 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,118 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,119 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,121 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,122 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,125 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,125 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,128 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,129 - ERROR - Error in session: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' +2025-12-25 10:37:54,131 - ERROR - Full traceback: +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1274, in wrapper + result = original_function(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4080, in completion + raise exception_type( + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 1243, in completion + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + ^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 455, in get_llm_provider + raise e + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py", line 432, in get_llm_provider + raise litellm.exceptions.BadRequestError( # type: ignore +litellm.exceptions.BadRequestError: litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=meta-llama/Llama-3.3-70B-Instruct + Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4094, in completion_with_retries + import tenacity +ModuleNotFoundError: No module named 'tenacity' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 201, in run_single_session + response = agent_adapter.generate_response(query, turns[:-1]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/reflection_grpo_adapter.py", line 97, in generate_response + response = self._agent.generate_collaborator_response(self._conversation_history) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 88, in generate_collaborator_response + conversation = self.add_scaffolding_to_conversation(conversation) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 71, in add_scaffolding_to_conversation + scaffolding_response = self.completion(scaffolding_messages) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/collaborativeagents/agents/collaborator_agent.py", line 50, in completion + response = llm_completion(model=self.model_name, messages=messages, num_retries=self.num_retries, **self.kwargs).choices[0].message.content + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/utils.py", line 1385, in wrapper + return litellm.completion_with_retries(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/litellm/main.py", line 4096, in completion_with_retries + raise Exception( +Exception: tenacity import failed please run `pip install tenacity`. ErrorNo module named 'tenacity' + +2025-12-25 10:37:54,141 - INFO - Report saved to ../results/collab_baselines_20251225_103724/20251225_103746/report.md diff --git a/collaborativeagents/slurm/logs/run_collab_baselines_14355975.err b/collaborativeagents/slurm/logs/run_collab_baselines_14355975.err new file mode 100644 index 0000000..c82b955 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_collab_baselines_14355975.err @@ -0,0 +1,70 @@ +2025-12-25 10:52:43,142 - INFO - Loaded dataset: math-500 +2025-12-25 10:52:43,385 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 10:52:43,386 - INFO - Running method: contextual +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.47s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.43s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.59s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +2025-12-25 10:53:08,251 - INFO - Profile 1/20 +2025-12-25 10:54:31,245 - INFO - Profile 2/20 +2025-12-25 10:55:49,021 - INFO - Profile 3/20 +2025-12-25 10:57:07,721 - INFO - Profile 4/20 +2025-12-25 10:57:46,873 - INFO - Profile 5/20 +2025-12-25 10:58:26,077 - INFO - Profile 6/20 +2025-12-25 10:59:05,298 - INFO - Profile 7/20 +2025-12-25 10:59:44,507 - INFO - Profile 8/20 +2025-12-25 11:00:23,700 - INFO - Profile 9/20 +2025-12-25 11:01:02,919 - INFO - Profile 10/20 +2025-12-25 11:01:42,131 - INFO - Profile 11/20 +2025-12-25 11:02:57,684 - INFO - Profile 12/20 +2025-12-25 11:03:56,775 - INFO - Profile 13/20 +2025-12-25 11:05:03,753 - INFO - Profile 14/20 +2025-12-25 11:05:51,697 - INFO - Profile 15/20 +2025-12-25 11:06:52,101 - INFO - Profile 16/20 +2025-12-25 11:07:27,746 - INFO - Profile 17/20 +2025-12-25 11:08:54,497 - INFO - Profile 18/20 +2025-12-25 11:10:40,288 - INFO - Profile 19/20 +2025-12-25 11:12:07,984 - INFO - Profile 20/20 +2025-12-25 11:12:44,173 - INFO - Running method: reflection +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.00s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:20<00:06, 6.78s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.87s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.49s/it] +2025-12-25 11:13:06,758 - INFO - Profile 1/20 +2025-12-25 11:14:12,362 - INFO - Profile 2/20 +2025-12-25 11:15:35,445 - INFO - Profile 3/20 +2025-12-25 11:16:54,467 - INFO - Profile 4/20 +2025-12-25 11:18:12,199 - INFO - Profile 5/20 +2025-12-25 11:19:42,312 - INFO - Profile 6/20 +2025-12-25 11:21:06,922 - INFO - Profile 7/20 +2025-12-25 11:22:54,996 - INFO - Profile 8/20 +2025-12-25 11:24:08,753 - INFO - Profile 9/20 +2025-12-25 11:25:19,792 - INFO - Profile 10/20 +2025-12-25 11:26:42,922 - INFO - Profile 11/20 +2025-12-25 11:28:16,287 - INFO - Profile 12/20 +2025-12-25 11:29:37,361 - INFO - Profile 13/20 +2025-12-25 11:30:51,174 - INFO - Profile 14/20 +2025-12-25 11:32:25,617 - INFO - Profile 15/20 +2025-12-25 11:33:26,202 - INFO - Profile 16/20 +2025-12-25 11:34:28,137 - INFO - Profile 17/20 +2025-12-25 11:35:45,693 - INFO - Profile 18/20 +2025-12-25 11:37:00,624 - INFO - Profile 19/20 +2025-12-25 11:38:12,427 - INFO - Profile 20/20 +2025-12-25 11:39:29,675 - INFO - Running method: reflection_grpo +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.18s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.71s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.35s/it] +2025-12-25 11:39:51,748 - INFO - Profile 1/20 +2025-12-25 11:41:06,412 - INFO - Profile 2/20 +2025-12-25 11:42:24,983 - INFO - Profile 3/20 +2025-12-25 11:43:42,715 - INFO - Profile 4/20 +2025-12-25 11:45:21,420 - INFO - Profile 5/20 +2025-12-25 11:46:41,875 - INFO - Profile 6/20 +2025-12-25 11:48:09,967 - INFO - Profile 7/20 +2025-12-25 11:49:25,792 - INFO - Profile 8/20 +2025-12-25 11:50:50,009 - INFO - Profile 9/20 +2025-12-25 11:52:26,201 - INFO - Profile 10/20 +2025-12-25 11:53:51,842 - INFO - Profile 11/20 +2025-12-25 11:55:20,284 - INFO - Profile 12/20 +2025-12-25 11:56:36,649 - INFO - Profile 13/20 +2025-12-25 11:58:06,896 - INFO - Profile 14/20 +2025-12-25 11:59:31,014 - INFO - Profile 15/20 +2025-12-25 12:00:48,734 - INFO - Profile 16/20 +2025-12-25 12:02:23,732 - INFO - Profile 17/20 +2025-12-25 12:03:20,783 - INFO - Profile 18/20 +2025-12-25 12:04:44,524 - INFO - Profile 19/20 +2025-12-25 12:06:08,471 - INFO - Profile 20/20 +2025-12-25 12:07:36,561 - INFO - Report saved to ../results/collab_baselines_20251225_105235/20251225_105243/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355787.err b/collaborativeagents/slurm/logs/run_expts_a100_14355787.err new file mode 100644 index 0000000..4845249 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355787.err @@ -0,0 +1,15 @@ +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 31, in <module> + from adapters.personalized_llm_adapter import PersonalizedLLMAdapter, create_baseline_adapter + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/__init__.py", line 3, in <module> + from .personalized_llm_adapter import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 21, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/__init__.py", line 5, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 35, in <module> + from personalization.models.reranker.qwen3_reranker import Qwen3Reranker + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/reranker/qwen3_reranker.py", line 26 + self.model = AutoModelForCausalLM.from_pretrained( + ^ +IndentationError: expected an indented block after 'else' statement on line 24 diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355797.err b/collaborativeagents/slurm/logs/run_expts_a100_14355797.err new file mode 100644 index 0000000..28ce959 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355797.err @@ -0,0 +1,14 @@ +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 31, in <module> + from adapters.personalized_llm_adapter import PersonalizedLLMAdapter, create_baseline_adapter + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/__init__.py", line 3, in <module> + from .personalized_llm_adapter import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 21, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/__init__.py", line 5, in <module> + from personalization.serving.personalized_llm import ( + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 38, in <module> + from personalization.user_model.features import ItemProjection + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/user_model/features.py", line 3, in <module> + from sklearn.decomposition import PCA +ModuleNotFoundError: No module named 'sklearn' diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355807.err b/collaborativeagents/slurm/logs/run_expts_a100_14355807.err new file mode 100644 index 0000000..33fa583 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355807.err @@ -0,0 +1,10 @@ +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 484, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 471, in main + runner = ExperimentRunner(config) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 99, in __init__ + self.conflict_generator = ConflictScenarioGenerator() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: ConflictScenarioGenerator.__init__() missing 1 required positional argument: 'profile' diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355816.err b/collaborativeagents/slurm/logs/run_expts_a100_14355816.err new file mode 100644 index 0000000..f147ab6 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355816.err @@ -0,0 +1,22 @@ +2025-12-25 07:53:36,898 - INFO - Loaded dataset: math-500 +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 484, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 471, in main + runner = ExperimentRunner(config) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 111, in __init__ + self.profiles = self._load_profiles() + ^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 117, in _load_profiles + profiles = json.load(f) + ^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/json/__init__.py", line 293, in load + return loads(fp.read(), + ^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/json/__init__.py", line 346, in loads + return _default_decoder.decode(s) + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/json/decoder.py", line 340, in decode + raise JSONDecodeError("Extra data", s, end) +json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 10782) diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355842.err b/collaborativeagents/slurm/logs/run_expts_a100_14355842.err new file mode 100644 index 0000000..0144ee9 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355842.err @@ -0,0 +1,23 @@ +2025-12-25 07:55:08,518 - INFO - Loaded dataset: math-500 +2025-12-25 07:55:08,527 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 07:55:08,528 - INFO - Running method: rag_vector +2025-12-25 07:55:08,528 - INFO - Profile 1/2 +
Generating test split: 0%| | 0/500 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 500/500 [00:00<00:00, 9910.65 examples/s] +2025-12-25 07:55:12,047 - ERROR - Error in session: No module named 'json_repair' +2025-12-25 07:55:12,050 - ERROR - Error in session: No module named 'json_repair' +2025-12-25 07:55:12,050 - INFO - Profile 2/2 +2025-12-25 07:55:12,053 - ERROR - Error in session: No module named 'json_repair' +2025-12-25 07:55:12,056 - ERROR - Error in session: No module named 'json_repair' +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 488, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 476, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 292, in run_all + analysis = self._analyze_results(all_results) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 365, in _analyze_results + best = max(values, key=values.get) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ValueError: max() arg is an empty sequence diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355851.err b/collaborativeagents/slurm/logs/run_expts_a100_14355851.err new file mode 100644 index 0000000..59bbe1a --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355851.err @@ -0,0 +1,28 @@ +2025-12-25 07:58:42,438 - INFO - Loaded dataset: math-500 +2025-12-25 07:58:42,447 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 07:58:42,448 - INFO - Running method: rag_vector +2025-12-25 07:58:42,448 - INFO - Profile 1/2 +2025-12-25 07:58:47,959 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,959 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,960 - INFO - Profile 2/2 +2025-12-25 07:58:47,960 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,960 - ERROR - Error in session: ConversationGenerator.__init__() got an unexpected keyword argument 'user_model' +2025-12-25 07:58:47,962 - WARNING - No values for metric task_success_rate, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_user_tokens, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_total_tokens, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_enforcement_count, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric avg_preference_compliance, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric conflict_resolution_accuracy, skipping comparison +2025-12-25 07:58:47,962 - WARNING - No values for metric over_personalization_rate, skipping comparison +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 491, in <module> + main() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 479, in main + analysis = runner.run_all() + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 299, in run_all + self._generate_report(analysis) + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 414, in _generate_report + best = analysis["comparison"][metric_key]["best_method"] + ~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +KeyError: 'task_success_rate' diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355856.err b/collaborativeagents/slurm/logs/run_expts_a100_14355856.err new file mode 100644 index 0000000..608e91f --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355856.err @@ -0,0 +1,10 @@ +2025-12-25 08:08:52,658 - INFO - Loaded dataset: math-500 +2025-12-25 08:08:52,698 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:08:52,699 - INFO - Running method: rag_vector +2025-12-25 08:08:52,699 - INFO - Profile 1/2 +2025-12-25 08:08:55,383 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,383 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,383 - INFO - Profile 2/2 +2025-12-25 08:08:55,384 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,384 - ERROR - Error in session: [Errno 2] No such file or directory: '/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/configs/local_models.yaml' +2025-12-25 08:08:55,390 - INFO - Report saved to ../results/test_a100_20251225_080844/20251225_080852/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355861.err b/collaborativeagents/slurm/logs/run_expts_a100_14355861.err new file mode 100644 index 0000000..fac726b --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355861.err @@ -0,0 +1,14 @@ +2025-12-25 08:12:25,638 - INFO - Loaded dataset: math-500 +2025-12-25 08:12:25,647 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:12:25,648 - INFO - Running method: rag_vector +2025-12-25 08:12:25,648 - INFO - Profile 1/2 +2025-12-25 08:12:28,020 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,062 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,062 - INFO - Profile 2/2 +2025-12-25 08:12:28,107 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,155 - ERROR - Error in session: models/qwen3-embedding-8b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=<your_token>` +2025-12-25 08:12:28,189 - INFO - Report saved to ../results/test_a100_20251225_081218/20251225_081225/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355863.err b/collaborativeagents/slurm/logs/run_expts_a100_14355863.err new file mode 100644 index 0000000..1a41e92 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355863.err @@ -0,0 +1,11 @@ +2025-12-25 08:15:58,390 - INFO - Loaded dataset: math-500 +2025-12-25 08:15:58,399 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:15:58,399 - INFO - Running method: rag_vector +2025-12-25 08:15:58,399 - INFO - Profile 1/2 +`torch_dtype` is deprecated! Use `dtype` instead! +2025-12-25 08:16:02,559 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:02,854 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:02,874 - INFO - Profile 2/2 +2025-12-25 08:16:03,126 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:03,394 - ERROR - Error in session: Using a `device_map`, `tp_plan`, `torch.device` context manager or setting `torch.set_default_device(device)` requires `accelerate`. You can install it with `pip install accelerate` +2025-12-25 08:16:03,459 - INFO - Report saved to ../results/test_a100_20251225_081551/20251225_081558/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355865.err b/collaborativeagents/slurm/logs/run_expts_a100_14355865.err new file mode 100644 index 0000000..769bc20 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355865.err @@ -0,0 +1,19 @@ +2025-12-25 08:20:02,816 - INFO - Loaded dataset: math-500 +2025-12-25 08:20:02,951 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:20:02,952 - INFO - Running method: rag_vector +2025-12-25 08:20:02,952 - INFO - Profile 1/2 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:22, 7.63s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:14<00:13, 6.92s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:06, 6.94s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.41s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:21, 5.36s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.40s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:16<00:10, 5.38s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:20<00:05, 5.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 4.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 4.58s/it] +2025-12-25 08:20:51,879 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.19s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.23s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.39s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.53s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.29s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.44s/it] +2025-12-25 08:21:14,611 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +2025-12-25 08:21:14,661 - INFO - Profile 2/2 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.22s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.33s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.08s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.45s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.42s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.17s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.73s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.16s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.39s/it] +2025-12-25 08:21:37,445 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.34s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.28s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.06s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.44s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.43s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.24s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.77s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.17s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.41s/it] +2025-12-25 08:22:00,232 - ERROR - Error in session: expected an indented block after 'else' statement on line 34 (llama_instruct.py, line 36) +2025-12-25 08:22:00,292 - INFO - Report saved to ../results/test_a100_20251225_081954/20251225_082002/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355871.err b/collaborativeagents/slurm/logs/run_expts_a100_14355871.err new file mode 100644 index 0000000..447cacb --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355871.err @@ -0,0 +1,16 @@ +2025-12-25 08:28:55,431 - INFO - Loaded dataset: math-500 +2025-12-25 08:28:55,625 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:28:55,626 - INFO - Running method: rag_vector +2025-12-25 08:28:55,626 - INFO - Profile 1/2 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.41s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.48s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 4.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:22<00:00, 5.62s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:09<00:36, 9.15s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:14<00:20, 6.98s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:21<00:13, 6.76s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:25<00:05, 5.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 4.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 5.61s/it] +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.54s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.88s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.22s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.34s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.28s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.39s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.70s/it] +2025-12-25 08:31:49,842 - INFO - Profile 2/2 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.69s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.07s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:07, 3.58s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.44s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.76s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.31s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.69s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.94s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.32s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:10, 5.23s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.33s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.50s/it] +2025-12-25 08:33:27,343 - INFO - Report saved to ../results/test_a100_20251225_082834/20251225_082855/report.md diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355878.err b/collaborativeagents/slurm/logs/run_expts_a100_14355878.err new file mode 100644 index 0000000..8634294 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355878.err @@ -0,0 +1,109 @@ +2025-12-25 08:39:20,163 - INFO - Loaded dataset: math-500 +2025-12-25 08:39:20,285 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:39:20,285 - INFO - Running method: vanilla +2025-12-25 08:39:20,285 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:10, 5.18s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.72s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.17s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.66s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.62s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.27s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.36s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.85s/it] +2025-12-25 08:40:03,460 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.11s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.15s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.14s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.57s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.07s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.72s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.21s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.32s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.11s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.66s/it] +2025-12-25 08:40:38,434 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.68s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.98s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.20s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.59s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:09, 4.61s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:04, 4.02s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.90s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.30s/it] +2025-12-25 08:41:11,060 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.94s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.15s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.14s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.57s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.06s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.58s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:08, 4.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.94s/it] +2025-12-25 08:41:42,373 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.36s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.22s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.10s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.54s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.07s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.34s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.91s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:11<00:08, 4.20s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.77s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.11s/it] +2025-12-25 08:42:14,703 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:42:14,780 - INFO - Profile 2/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.99s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.14s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.12s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.54s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:08, 4.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.73s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +2025-12-25 08:42:46,213 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.86s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.18s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.22s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.62s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.10s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:08, 2.18s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.01s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.75s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.53s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.83s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.04s/it] +2025-12-25 08:43:19,397 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.68s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.52s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.67s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.45s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.85s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.92s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.52s/it] +2025-12-25 08:43:48,538 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.34s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.81s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.73s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.94s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.98s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:07, 3.59s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.34s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.89s/it] +2025-12-25 08:44:18,865 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.03s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.63s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.29s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.44s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.09s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.76s/it] +2025-12-25 08:44:48,291 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:44:48,358 - INFO - Profile 3/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.77s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.55s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.70s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.38s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.52s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.60s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.24s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.58s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.90s/it] +2025-12-25 08:45:18,280 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.91s/it]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.09s/it] +2025-12-25 08:45:24,840 - ERROR - Error in session: CUDA out of memory. Tried to allocate 96.00 MiB. GPU 2 has a total capacity of 39.49 GiB of which 90.31 MiB is free. Including non-PyTorch memory, this process has 39.40 GiB memory in use. Of the allocated memory 38.90 GiB is allocated by PyTorch, and 99.47 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.61s/it]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.83s/it] +2025-12-25 08:45:31,127 - ERROR - Error in session: CUDA out of memory. Tried to allocate 96.00 MiB. GPU 1 has a total capacity of 39.49 GiB of which 8.31 MiB is free. Including non-PyTorch memory, this process has 39.48 GiB memory in use. Of the allocated memory 38.89 GiB is allocated by PyTorch, and 181.97 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.57s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.38s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.24s/it] +2025-12-25 08:45:40,072 - ERROR - Error in session: CUDA out of memory. Tried to allocate 96.00 MiB. GPU 1 has a total capacity of 39.49 GiB of which 8.31 MiB is free. Including non-PyTorch memory, this process has 39.48 GiB memory in use. Of the allocated memory 38.89 GiB is allocated by PyTorch, and 181.97 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.61s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.51s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.60s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.65s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:17, 5.73s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.17s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.20s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.19s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.79s/it] +2025-12-25 08:46:14,167 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:46:14,242 - INFO - Profile 4/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.46s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.72s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.74s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.28s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:17, 5.84s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.20s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.31s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.88s/it] +2025-12-25 08:46:48,967 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.93s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.46s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.91s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.99s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.47s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.92s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.41s/it] +2025-12-25 08:47:21,876 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.67s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:09, 4.71s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.36s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.70s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.23s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:16, 5.51s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.85s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.00s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.59s/it] +2025-12-25 08:47:56,567 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.83s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.83s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.44s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.75s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.30s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.56s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.71s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.03s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.64s/it] +2025-12-25 08:48:31,945 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.19s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:09, 4.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.40s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.72s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.27s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.87s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.29s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.83s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.29s/it] +2025-12-25 08:49:05,843 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:49:05,907 - INFO - Profile 5/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:09, 4.59s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.19s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.77s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.49s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.74s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.00s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.42s/it] +2025-12-25 08:49:39,852 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.79s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.78s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.93s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.89s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:08, 2.12s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.37s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.63s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.36s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.65s/it] +2025-12-25 08:50:14,293 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.02s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.70s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.19s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.04s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.66s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.82s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.95s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.03s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.51s/it] +2025-12-25 08:50:48,033 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.00s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.64s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.95s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.46s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.89s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.73s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.68s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.94s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.43s/it] +2025-12-25 08:51:20,931 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.69s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.68s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.03s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.92s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.73s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.92s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.56s/it] +2025-12-25 08:51:54,525 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:51:54,585 - INFO - Profile 6/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.24s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.61s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.61s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:15, 5.24s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.23s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.25s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.77s/it] +2025-12-25 08:52:27,892 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.12s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.10s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.64s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.12s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.80s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.74s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.03s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.15s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.59s/it] +2025-12-25 08:53:02,452 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.93s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.93s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:12, 4.17s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.77s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.94s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.33s/it] +2025-12-25 08:53:35,074 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.10s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.82s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.13s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.02s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.67s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.44s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.87s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.33s/it] +2025-12-25 08:54:08,083 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.08s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:08, 4.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.04s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 3.00s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.33s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.69s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.77s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.18s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.24s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.68s/it] +2025-12-25 08:54:42,574 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 08:54:42,637 - INFO - Profile 7/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.87s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.39s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.44s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.75s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.25s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.57s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:09, 4.58s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.43s/it] +2025-12-25 08:55:16,985 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.10s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.86s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.26s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.92s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:15, 3.79s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.50s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.13s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.19s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.16s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.85s/it] +2025-12-25 08:55:56,547 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.74s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.80s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.11s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.55s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.24s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.60s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.32s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.34s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 4.18s/it] +2025-12-25 08:56:33,918 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.98s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.82s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:04, 4.10s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.55s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 3.00s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.24s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.43s/it][2025-12-25T08:56:58.364] error: *** JOB 14355878 ON gpua055 CANCELLED AT 2025-12-25T08:56:58 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355885.err b/collaborativeagents/slurm/logs/run_expts_a100_14355885.err new file mode 100644 index 0000000..ffc2040 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355885.err @@ -0,0 +1,17 @@ +2025-12-25 08:58:07,929 - INFO - Loaded dataset: math-500 +2025-12-25 08:58:08,121 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 08:58:08,122 - INFO - Running method: vanilla +2025-12-25 08:58:08,122 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.19s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.10s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:09<00:00, 2.46s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:14, 3.68s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:16, 5.46s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.22s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.30s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.91s/it] +2025-12-25 08:58:47,629 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.44s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.11s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.27s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.65s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:13, 3.45s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:15, 5.12s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.65s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.00s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.10s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.67s/it] +2025-12-25 08:59:23,124 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.95s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.41s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:15, 5.30s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.86s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.12s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.11s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.68s/it] +2025-12-25 08:59:57,630 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.82s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.87s/it][2025-12-25T09:00:23.259] error: *** JOB 14355885 ON gpua055 CANCELLED AT 2025-12-25T09:00:23 DUE to SIGNAL Terminated *** +
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.74s/it]
\ No newline at end of file diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355888.err b/collaborativeagents/slurm/logs/run_expts_a100_14355888.err new file mode 100644 index 0000000..c5abe31 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355888.err @@ -0,0 +1,19 @@ +2025-12-25 09:02:11,022 - INFO - Loaded dataset: math-500 +2025-12-25 09:02:11,141 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:02:11,142 - INFO - Running method: vanilla +2025-12-25 09:02:11,142 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.63s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.64s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.83s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.71s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:10, 5.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.38s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.78s/it] +2025-12-25 09:02:50,489 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.60s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.88s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:09, 2.48s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:14, 4.72s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.85s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:04, 4.05s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.15s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.63s/it] +2025-12-25 09:03:24,392 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.32s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.79s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.62s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.90s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.34s/it] +2025-12-25 09:03:57,453 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.46s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.87s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.79s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.26s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:13, 4.61s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.80s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.28s/it] +2025-12-25 09:04:31,571 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.83s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.43s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.86s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.40s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.81s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.85s/it][2025-12-25T09:04:49.962] error: *** JOB 14355888 ON gpua055 CANCELLED AT 2025-12-25T09:04:49 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355896.err b/collaborativeagents/slurm/logs/run_expts_a100_14355896.err new file mode 100644 index 0000000..1f92e5d --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355896.err @@ -0,0 +1,12 @@ +2025-12-25 09:10:36,065 - INFO - Loaded dataset: math-500 +2025-12-25 09:10:36,277 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:10:36,278 - INFO - Running method: vanilla +2025-12-25 09:10:36,278 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.40s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.74s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:20<00:06, 6.83s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:07<00:30, 7.52s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:13<00:19, 6.37s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:19<00:13, 6.51s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:24<00:05, 5.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:27<00:00, 4.71s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:27<00:00, 5.48s/it] +2025-12-25 09:11:36,637 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.37s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.27s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.55s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.21s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.58s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.95s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.02s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.48s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.47s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.81s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.97s/it] +2025-12-25 09:12:05,675 - ERROR - Error in session: 'NoneType' object is not subscriptable +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.75s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.40s/it][2025-12-25T09:12:15.828] error: *** JOB 14355896 ON gpua036 CANCELLED AT 2025-12-25T09:12:15 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355901.err b/collaborativeagents/slurm/logs/run_expts_a100_14355901.err new file mode 100644 index 0000000..4ce1958 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355901.err @@ -0,0 +1,47 @@ +2025-12-25 09:12:44,146 - INFO - Loaded dataset: math-500 +2025-12-25 09:12:44,269 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:12:44,270 - INFO - Running method: vanilla +2025-12-25 09:12:44,270 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:07, 2.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.16s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.19s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.54s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.97s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.61s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.94s/it] +2025-12-25 09:13:17,230 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 09:13:17,235 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 124, in get_preference_extractor + model_path=spec_dict["local_path"], + ~~~~~~~~~^^^^^^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable + +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.20s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.59s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.89s/it] +2025-12-25 09:13:47,069 - ERROR - Error in session: 'NoneType' object is not subscriptable +2025-12-25 09:13:47,070 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 124, in get_preference_extractor + model_path=spec_dict["local_path"], + ~~~~~~~~~^^^^^^^^^^^^^^ +TypeError: 'NoneType' object is not subscriptable + +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.87s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:07, 3.57s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.53s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.20s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.64s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.21s/it][2025-12-25T09:14:07.340] error: *** JOB 14355901 ON gpua036 CANCELLED AT 2025-12-25T09:14:07 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355902.err b/collaborativeagents/slurm/logs/run_expts_a100_14355902.err new file mode 100644 index 0000000..c0437af --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355902.err @@ -0,0 +1,185 @@ +2025-12-25 09:14:36,278 - INFO - Loaded dataset: math-500 +2025-12-25 09:14:36,576 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:14:36,577 - INFO - Running method: vanilla +2025-12-25 09:14:36,577 - INFO - Profile 1/20 +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.14s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.32s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:05, 5.91s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 3.76s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.62s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:06<00:25, 6.47s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.69s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.16s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.17s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 3.31s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:20<00:00, 4.01s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:09, 3.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.29s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.54s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:07, 2.66s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.96s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:10<00:02, 2.60s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.05s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:11<00:00, 2.34s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.16s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.75s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.38s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:13<00:00, 3.32s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.54s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:07, 2.66s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.99s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.20s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.47s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:19, 6.65s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.64s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:14<00:04, 4.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.11s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.86s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.95s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.20s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.08s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.37s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.62s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:07, 7.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.04s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:07, 2.61s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.34s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.54s/it] +2025-12-25 09:18:54,429 - INFO - Profile 2/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:06, 3.47s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.58s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.91s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.42s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.86s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.62s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.43s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.96s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.26s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.45s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 3.00s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.78s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.12s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.80s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.19s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.50s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.98s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.80s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.87s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.46s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.97s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.80s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.73s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.51s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.01s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.73s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.83s/it] +2025-12-25 09:22:06,817 - ERROR - Error in session: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.77 GiB is allocated by PyTorch, and 210.60 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +2025-12-25 09:22:06,836 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 123, in get_preference_extractor + return QwenRuleExtractor( + ^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/preference_extractor/rule_extractor.py", line 36, in __init__ + self.model = AutoModelForCausalLM.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 604, in from_pretrained + return model_class.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 277, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5048, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5468, in _load_pretrained_model + _error_msgs, disk_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 843, in load_shard_file + disk_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 770, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + ^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.77 GiB is allocated by PyTorch, and 210.60 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) + +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.61s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.05s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.34s/it] +2025-12-25 09:22:24,976 - INFO - Based on the current allocation process, no modules could be assigned to the following devices due to insufficient memory: + - 0: 2484944896 bytes required +These minimum requirements are specific to this allocation attempt and may vary. Consider increasing the available memory for these devices to at least the specified minimum, or adjusting the model config. +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.51s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.98s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.65s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.79s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.86s/it] +2025-12-25 09:22:39,682 - INFO - Based on the current allocation process, no modules could be assigned to the following devices due to insufficient memory: + - 0: 560343040 bytes required +These minimum requirements are specific to this allocation attempt and may vary. Consider increasing the available memory for these devices to at least the specified minimum, or adjusting the model config. +2025-12-25 09:22:44,762 - ERROR - Error in session: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.76 GiB is allocated by PyTorch, and 219.26 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +2025-12-25 09:22:44,764 - ERROR - Full traceback: +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/run_experiments.py", line 192, in run_single_session + agent_adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 318, in _load_models + self._extractor = get_preference_extractor("rule") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/config/registry.py", line 123, in get_preference_extractor + return QwenRuleExtractor( + ^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/preference_extractor/rule_extractor.py", line 36, in __init__ + self.model = AutoModelForCausalLM.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 604, in from_pretrained + return model_class.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 277, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5048, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5468, in _load_pretrained_model + _error_msgs, disk_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 843, in load_shard_file + disk_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 770, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + ^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 39.49 GiB of which 12.31 MiB is free. Including non-PyTorch memory, this process has 39.47 GiB memory in use. Of the allocated memory 38.76 GiB is allocated by PyTorch, and 219.26 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) + +2025-12-25 09:22:44,854 - INFO - Profile 3/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.75s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.21s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.97s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.30s/it] +2025-12-25 09:23:02,714 - INFO - Based on the current allocation process, no modules could be assigned to the following devices due to insufficient memory: + - 0: 2484944896 bytes required +These minimum requirements are specific to this allocation attempt and may vary. Consider increasing the available memory for these devices to at least the specified minimum, or adjusting the model config. +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.44s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.02s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.29s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.70s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.07s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.80s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.35s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.44s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.95s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.44s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.26s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.85s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.77s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.68s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.09s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.57s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.01s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.51s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.78s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.06s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.40s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.01s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.31s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.51s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.67s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.90s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 6.00s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.69s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.17s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.90s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:09, 3.15s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.32s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.04s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.41s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.67s/it] +2025-12-25 09:26:54,931 - INFO - Profile 4/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.57s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.94s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.14s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.93s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.34s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.29s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.56s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.85s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.20s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.13s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.77s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.29s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.79s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:06, 3.17s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:03, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:13<00:00, 2.62s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.35s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.83s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.63s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.08s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:04<00:07, 2.49s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.94s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:11<00:02, 2.91s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:12<00:00, 2.45s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.14s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.83s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:05, 5.91s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.64s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.08s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:01<00:07, 1.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.42s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:07, 3.63s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.52s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.74s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 3.00s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.15s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.02s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.16s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:15, 3.79s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.92s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:11<00:07, 3.98s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 2.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:17<00:00, 3.40s/it] +2025-12-25 09:30:47,307 - INFO - Profile 5/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.89s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.52s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.59s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.84s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.73s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.24s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.34s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.53s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.72s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.97s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.26s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.86s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.33s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.84s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.03s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.56s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.88s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.23s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.01s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.46s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.98s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.41s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.82s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.32s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:16, 5.54s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:08, 4.42s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.90s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.34s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.68s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.29s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:04, 4.12s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.13s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.05s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.68s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.26s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.82s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.24s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.07s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.93s/it] +2025-12-25 09:34:45,922 - INFO - Profile 6/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.60s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 4.00s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:07, 7.07s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.34s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.77s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.66s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.12s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.72s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.90s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.50s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.56s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.52s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.88s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.59s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.39s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.19s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.67s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.89s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.60s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.01s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.64s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 3.00s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.99s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.62s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.53s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.81s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.56s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.05s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.73s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.14s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.58s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.83s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:05<00:08, 2.89s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:08<00:05, 2.93s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.56s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.83s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.79s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.23s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.83s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.32s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.74s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:06, 3.38s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.99s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.76s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.14s/it] +2025-12-25 09:38:42,663 - INFO - Profile 7/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.81s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.75s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.03s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.11s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.91s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.41s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.14s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.71s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.58s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.93s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.18s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.02s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.84s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.28s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.57s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.28s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.78s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.63s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.99s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.38s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:07, 3.96s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.19s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.81s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.26s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:12, 3.15s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.43s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.20s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.68s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.03s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.46s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.07s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.25s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.32s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.89s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.11s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.58s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.50s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.10s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.85s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.21s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.24s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.88s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.32s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.91s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.72s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.02s/it] +2025-12-25 09:42:41,214 - INFO - Profile 8/20 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.93s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.22s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.97s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.48s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.81s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.60s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.26s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.66s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.71s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.03s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.27s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.33s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.22s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.83s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.39s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.86s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.25s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.07s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.54s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.47s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.82s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.29s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.32s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.65s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.09s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.55s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.89s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.27s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.11s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.59s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.51s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.85s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.95s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.73s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:16<00:06, 6.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.89s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.28s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.70s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.19s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:09<00:06, 3.09s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.64s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.54s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.86s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.55s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.90s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.69s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:11, 2.78s/it][2025-12-25T09:46:14.933] error: *** JOB 14355902 ON gpua065 CANCELLED AT 2025-12-25T09:46:14 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_expts_a100_14355919.err b/collaborativeagents/slurm/logs/run_expts_a100_14355919.err new file mode 100644 index 0000000..0a2aab0 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_expts_a100_14355919.err @@ -0,0 +1,97 @@ +2025-12-25 09:48:42,727 - INFO - Loaded dataset: math-500 +2025-12-25 09:48:42,889 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 09:48:42,890 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.41s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.19s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:06, 6.44s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 3.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:17<00:00, 4.43s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:02<00:10, 2.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:10, 3.40s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:06, 3.41s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:14<00:03, 3.87s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 2.84s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:15<00:00, 3.12s/it] +2025-12-25 09:49:25,411 - INFO - Profile 1/20 +2025-12-25 09:50:33,069 - INFO - Profile 2/20 +2025-12-25 09:51:32,211 - INFO - Profile 3/20 +2025-12-25 09:52:38,264 - INFO - Profile 4/20 +2025-12-25 09:53:45,537 - INFO - Profile 5/20 +2025-12-25 09:54:50,435 - INFO - Profile 6/20 +2025-12-25 09:55:54,476 - INFO - Profile 7/20 +2025-12-25 09:56:57,290 - INFO - Profile 8/20 +2025-12-25 09:57:59,489 - INFO - Profile 9/20 +2025-12-25 09:59:06,837 - INFO - Profile 10/20 +2025-12-25 10:00:16,031 - INFO - Profile 11/20 +2025-12-25 10:01:22,406 - INFO - Profile 12/20 +2025-12-25 10:02:29,213 - INFO - Profile 13/20 +2025-12-25 10:03:33,203 - INFO - Profile 14/20 +2025-12-25 10:04:37,973 - INFO - Profile 15/20 +2025-12-25 10:05:48,613 - INFO - Profile 16/20 +2025-12-25 10:06:55,828 - INFO - Profile 17/20 +2025-12-25 10:08:00,583 - INFO - Profile 18/20 +2025-12-25 10:09:08,100 - INFO - Profile 19/20 +2025-12-25 10:10:14,307 - INFO - Profile 20/20 +2025-12-25 10:11:19,207 - INFO - Running method: all_memory +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.04s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.63s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.27s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.08s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:21, 5.37s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:10<00:15, 5.16s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:16<00:10, 5.40s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:20<00:04, 4.91s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 3.86s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:22<00:00, 4.45s/it] +2025-12-25 10:12:53,238 - INFO - Profile 1/20 +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-25 10:13:59,767 - INFO - Profile 2/20 +2025-12-25 10:15:08,683 - INFO - Profile 3/20 +2025-12-25 10:16:16,674 - INFO - Profile 4/20 +2025-12-25 10:17:27,036 - INFO - Profile 5/20 +2025-12-25 10:18:36,939 - INFO - Profile 6/20 +2025-12-25 10:19:50,147 - INFO - Profile 7/20 +2025-12-25 10:20:58,377 - INFO - Profile 8/20 +2025-12-25 10:22:04,174 - INFO - Profile 9/20 +2025-12-25 10:23:04,466 - INFO - Profile 10/20 +2025-12-25 10:23:59,551 - INFO - Profile 11/20 +2025-12-25 10:25:00,817 - INFO - Profile 12/20 +2025-12-25 10:26:12,175 - INFO - Profile 13/20 +2025-12-25 10:27:07,257 - INFO - Profile 14/20 +2025-12-25 10:28:21,198 - INFO - Profile 15/20 +2025-12-25 10:29:25,304 - INFO - Profile 16/20 +2025-12-25 10:30:33,063 - INFO - Profile 17/20 +2025-12-25 10:31:41,527 - INFO - Profile 18/20 +2025-12-25 10:32:46,068 - INFO - Profile 19/20 +2025-12-25 10:33:55,833 - INFO - Profile 20/20 +2025-12-25 10:34:52,399 - INFO - Running method: rag +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.92s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.25s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.47s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.99s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:07<00:31, 7.92s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:13<00:19, 6.53s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:19<00:12, 6.32s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:24<00:05, 5.94s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:26<00:00, 4.46s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:26<00:00, 5.35s/it] +2025-12-25 10:36:26,619 - INFO - Profile 1/20 +2025-12-25 10:37:35,693 - INFO - Profile 2/20 +2025-12-25 10:38:41,748 - INFO - Profile 3/20 +2025-12-25 10:39:47,539 - INFO - Profile 4/20 +2025-12-25 10:40:52,966 - INFO - Profile 5/20 +2025-12-25 10:41:55,106 - INFO - Profile 6/20 +2025-12-25 10:43:03,681 - INFO - Profile 7/20 +2025-12-25 10:44:11,999 - INFO - Profile 8/20 +2025-12-25 10:45:27,468 - INFO - Profile 9/20 +2025-12-25 10:46:29,111 - INFO - Profile 10/20 +2025-12-25 10:47:36,526 - INFO - Profile 11/20 +2025-12-25 10:48:40,778 - INFO - Profile 12/20 +2025-12-25 10:49:56,738 - INFO - Profile 13/20 +2025-12-25 10:51:03,796 - INFO - Profile 14/20 +2025-12-25 10:52:10,335 - INFO - Profile 15/20 +2025-12-25 10:53:16,174 - INFO - Profile 16/20 +2025-12-25 10:54:21,228 - INFO - Profile 17/20 +2025-12-25 10:55:26,718 - INFO - Profile 18/20 +2025-12-25 10:56:33,006 - INFO - Profile 19/20 +2025-12-25 10:57:36,404 - INFO - Profile 20/20 +2025-12-25 10:58:45,324 - INFO - Running method: rag_vector +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:22, 7.43s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:13<00:13, 6.65s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:21<00:07, 7.12s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.58s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.48s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:07<00:30, 7.59s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:13<00:20, 6.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:19<00:12, 6.48s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:25<00:06, 6.23s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 4.78s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:28<00:00, 5.60s/it] +2025-12-25 10:59:46,496 - INFO - Profile 1/20 +2025-12-25 11:00:34,992 - INFO - Profile 2/20 +2025-12-25 11:01:23,029 - INFO - Profile 3/20 +2025-12-25 11:02:15,132 - INFO - Profile 4/20 +2025-12-25 11:02:57,063 - INFO - Profile 5/20 +2025-12-25 11:03:49,132 - INFO - Profile 6/20 +2025-12-25 11:04:41,128 - INFO - Profile 7/20 +2025-12-25 11:05:28,179 - INFO - Profile 8/20 +2025-12-25 11:06:17,236 - INFO - Profile 9/20 +2025-12-25 11:07:04,377 - INFO - Profile 10/20 +2025-12-25 11:07:55,209 - INFO - Profile 11/20 +2025-12-25 11:08:41,927 - INFO - Profile 12/20 +2025-12-25 11:09:33,573 - INFO - Profile 13/20 +2025-12-25 11:10:20,629 - INFO - Profile 14/20 +2025-12-25 11:11:05,039 - INFO - Profile 15/20 +2025-12-25 11:11:57,558 - INFO - Profile 16/20 +2025-12-25 11:12:45,379 - INFO - Profile 17/20 +2025-12-25 11:13:30,642 - INFO - Profile 18/20 +2025-12-25 11:14:22,279 - INFO - Profile 19/20 +2025-12-25 11:15:11,250 - INFO - Profile 20/20 +2025-12-25 11:16:03,557 - INFO - Report saved to ../results/full_20251225_094835/20251225_094842/report.md diff --git a/collaborativeagents/slurm/logs/run_multiturn_14357110.err b/collaborativeagents/slurm/logs/run_multiturn_14357110.err new file mode 100644 index 0000000..53553f1 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_multiturn_14357110.err @@ -0,0 +1,9 @@ +2025-12-25 21:55:51,030 - INFO - Loaded dataset: math-500 +2025-12-25 21:55:51,242 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 21:55:51,243 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:15, 5.28s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.08s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.50s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.12s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:18, 4.75s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:13, 4.55s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.55s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:04, 4.26s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.38s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.87s/it] +2025-12-25 21:56:39,760 - INFO - Profile 1/5 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:18, 6.03s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.01s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 5.80s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.43s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.98s/it] +[2025-12-25T21:58:59.259] error: *** JOB 14357110 ON gpua016 CANCELLED AT 2025-12-25T21:58:59 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/run_multiturn_14357122.err b/collaborativeagents/slurm/logs/run_multiturn_14357122.err new file mode 100644 index 0000000..b79f0e7 --- /dev/null +++ b/collaborativeagents/slurm/logs/run_multiturn_14357122.err @@ -0,0 +1,98 @@ +2025-12-25 22:08:21,888 - INFO - Loaded dataset: math-500 +2025-12-25 22:08:22,033 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-25 22:08:22,034 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:14, 4.85s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.08s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.69s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.60s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:03<00:15, 3.81s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:06<00:09, 3.25s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:10<00:06, 3.45s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:12<00:03, 3.07s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.49s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.86s/it] +2025-12-25 22:08:59,678 - INFO - Profile 1/5 +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:10<00:32, 10.97s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:15<00:14, 7.08s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:05, 5.68s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.06s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.23s/it] +2025-12-25 22:12:43,460 - WARNING - User agent failed to respond at turn 4 +2025-12-25 22:14:39,792 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:14:39,793 - INFO - Profile 2/5 +2025-12-25 22:17:30,565 - INFO - Profile 3/5 +2025-12-25 22:20:00,571 - INFO - Profile 4/5 +2025-12-25 22:23:05,146 - WARNING - User agent failed to respond at turn 4 +2025-12-25 22:23:35,365 - INFO - Profile 5/5 +2025-12-25 22:26:59,994 - INFO - Running method: all_memory +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:08<00:24, 8.06s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:10, 5.48s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:20<00:07, 7.15s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 4.51s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:21<00:00, 5.33s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:22, 5.64s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:12, 4.14s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:10, 5.08s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:17<00:03, 3.98s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 2.97s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.68s/it] +2025-12-25 22:28:23,340 - INFO - Profile 1/5 +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. +2025-12-25 22:29:58,580 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:30:39,359 - INFO - Profile 2/5 +2025-12-25 22:34:23,370 - INFO - Profile 3/5 +2025-12-25 22:35:04,289 - WARNING - User agent failed to respond at turn 2 +2025-12-25 22:35:30,064 - WARNING - User agent failed to respond at turn 2 +2025-12-25 22:36:33,412 - WARNING - User agent failed to respond at turn 6 +2025-12-25 22:36:33,412 - INFO - Profile 4/5 +2025-12-25 22:38:38,658 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:39:23,955 - INFO - Profile 5/5 +2025-12-25 22:42:19,402 - INFO - Running method: rag +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:06<00:20, 6.89s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.09s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.42s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.16s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.98s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:06<00:24, 6.16s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:11<00:16, 5.55s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:17<00:11, 5.90s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:22<00:05, 5.37s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:23<00:00, 4.06s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:23<00:00, 4.78s/it] +2025-12-25 22:43:31,077 - INFO - Profile 1/5 +2025-12-25 22:46:19,125 - INFO - Profile 2/5 +2025-12-25 22:49:18,368 - INFO - Profile 3/5 +2025-12-25 22:53:00,495 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:53:00,497 - INFO - Profile 4/5 +2025-12-25 22:54:01,784 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:58:07,157 - INFO - Profile 5/5 +2025-12-25 22:58:54,351 - WARNING - User agent failed to respond at turn 3 +2025-12-25 22:59:40,507 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:00:10,569 - INFO - Running method: rag_vector +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:22, 7.37s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:10<00:10, 5.14s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 3.96s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.74s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:06<00:25, 6.35s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:12, 4.31s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:15<00:10, 5.22s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.18s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.16s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.89s/it] +2025-12-25 23:01:19,774 - INFO - Profile 1/5 +2025-12-25 23:03:56,207 - INFO - Profile 2/5 +2025-12-25 23:06:30,341 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:06:30,342 - INFO - Profile 3/5 +2025-12-25 23:09:50,352 - WARNING - User agent failed to respond at turn 7 +2025-12-25 23:11:12,291 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:11:12,293 - INFO - Profile 4/5 +2025-12-25 23:14:00,507 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:15:21,185 - INFO - Profile 5/5 +2025-12-25 23:17:09,189 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:17:38,489 - INFO - Running method: contextual +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.83s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:11, 5.97s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:17<00:05, 5.85s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.31s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.89s/it] +2025-12-25 23:17:58,530 - INFO - Profile 1/5 +2025-12-25 23:22:15,614 - WARNING - User agent failed to respond at turn 5 +2025-12-25 23:23:01,495 - INFO - Profile 2/5 +2025-12-25 23:26:21,325 - WARNING - User agent failed to respond at turn 12 +2025-12-25 23:26:21,326 - INFO - Profile 3/5 +2025-12-25 23:29:17,191 - WARNING - User agent failed to respond at turn 7 +2025-12-25 23:30:41,180 - INFO - Profile 4/5 +2025-12-25 23:31:24,578 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:33:26,694 - WARNING - User agent failed to respond at turn 6 +2025-12-25 23:35:25,025 - WARNING - User agent failed to respond at turn 6 +2025-12-25 23:35:25,025 - INFO - Profile 5/5 +2025-12-25 23:36:14,963 - WARNING - User agent failed to respond at turn 3 +2025-12-25 23:37:33,084 - WARNING - User agent failed to respond at turn 3 +2025-12-25 23:38:11,089 - INFO - Running method: reflection +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.99s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.61s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.32s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.35s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.87s/it] +2025-12-25 23:38:27,190 - INFO - Profile 1/5 +2025-12-25 23:41:28,520 - WARNING - User agent failed to respond at turn 3 +2025-12-25 23:42:37,103 - INFO - Profile 2/5 +2025-12-25 23:46:33,054 - WARNING - User agent failed to respond at turn 7 +2025-12-25 23:46:46,658 - INFO - Profile 3/5 +2025-12-25 23:49:40,906 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:50:58,786 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:51:12,246 - INFO - Profile 4/5 +2025-12-25 23:52:14,159 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:55:01,535 - WARNING - User agent failed to respond at turn 4 +2025-12-25 23:56:57,317 - INFO - Profile 5/5 +2025-12-25 23:58:27,891 - WARNING - User agent failed to respond at turn 2 +2025-12-25 23:59:29,746 - INFO - Running method: reflection_grpo +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:07<00:21, 7.21s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:12<00:12, 6.18s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:05, 5.82s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.28s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:19<00:00, 4.99s/it] +2025-12-25 23:59:50,260 - INFO - Profile 1/5 +2025-12-26 00:00:31,696 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:03:33,202 - INFO - Profile 2/5 +2025-12-26 00:06:53,817 - INFO - Profile 3/5 +2025-12-26 00:10:53,169 - WARNING - User agent failed to respond at turn 4 +2025-12-26 00:12:53,034 - WARNING - User agent failed to respond at turn 4 +2025-12-26 00:13:06,491 - INFO - Profile 4/5 +2025-12-26 00:13:59,355 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:18:16,345 - INFO - Profile 5/5 +2025-12-26 00:18:53,569 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:19:48,324 - WARNING - User agent failed to respond at turn 2 +2025-12-26 00:20:53,392 - WARNING - User agent failed to respond at turn 3 +2025-12-26 00:21:06,861 - INFO - Report saved to ../results/multiturn_test_20251225_220813/20251225_220821/report.md diff --git a/collaborativeagents/slurm/logs/test_70b_14357753.err b/collaborativeagents/slurm/logs/test_70b_14357753.err new file mode 100644 index 0000000..994c585 --- /dev/null +++ b/collaborativeagents/slurm/logs/test_70b_14357753.err @@ -0,0 +1 @@ +`torch_dtype` is deprecated! Use `dtype` instead! diff --git a/collaborativeagents/slurm/logs/test_70b_14357762.err b/collaborativeagents/slurm/logs/test_70b_14357762.err new file mode 100644 index 0000000..db6b275 --- /dev/null +++ b/collaborativeagents/slurm/logs/test_70b_14357762.err @@ -0,0 +1,24 @@ +`torch_dtype` is deprecated! Use `dtype` instead! +
Fetching 9 files: 0%| | 0/9 [00:00<?, ?it/s]
Fetching 9 files: 11%|█ | 1/9 [01:04<08:38, 64.82s/it]
Fetching 9 files: 22%|██▏ | 2/9 [01:05<03:09, 27.05s/it]
Fetching 9 files: 44%|████▍ | 4/9 [01:06<00:52, 10.45s/it]
Fetching 9 files: 78%|███████▊ | 7/9 [01:33<00:19, 9.67s/it]
Fetching 9 files: 100%|██████████| 9/9 [01:33<00:00, 10.43s/it] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:03<00:26, 3.34s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:09<00:33, 4.81s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:11<00:22, 3.73s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:17<00:22, 4.51s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [00:23<00:20, 5.10s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [00:28<00:15, 5.08s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:23<00:43, 21.52s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:26<00:15, 15.40s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:27<00:00, 10.90s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:27<00:00, 9.69s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.90s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:11, 5.91s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:19<00:06, 6.56s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 4.39s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:20<00:00, 5.05s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:15<01:02, 15.59s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:21<00:30, 10.08s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:27<00:16, 8.23s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:32<00:06, 6.79s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:34<00:00, 5.01s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:34<00:00, 6.85s/it] +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:02<00:19, 2.47s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [01:00<04:05, 35.01s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [01:05<02:07, 21.33s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [01:10<01:14, 14.81s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [01:15<00:45, 11.36s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:20<00:27, 9.25s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:30<00:19, 9.50s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:32<00:07, 7.02s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:33<00:00, 5.34s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:33<00:00, 10.43s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.44s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:09, 4.72s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:15<00:05, 5.15s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.30s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:15<00:00, 3.87s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:05<00:21, 5.25s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:13, 4.63s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:13<00:09, 4.53s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:16<00:03, 3.87s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.05s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:18<00:00, 3.66s/it] diff --git a/collaborativeagents/slurm/logs/test_extractor_14363568.err b/collaborativeagents/slurm/logs/test_extractor_14363568.err new file mode 100644 index 0000000..9b05d44 --- /dev/null +++ b/collaborativeagents/slurm/logs/test_extractor_14363568.err @@ -0,0 +1,3 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details. diff --git a/collaborativeagents/slurm/logs/test_multiturn_14357116.err b/collaborativeagents/slurm/logs/test_multiturn_14357116.err new file mode 100644 index 0000000..91eba3b --- /dev/null +++ b/collaborativeagents/slurm/logs/test_multiturn_14357116.err @@ -0,0 +1,13 @@ +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:10, 3.54s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.66s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.54s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 2.71s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:12<00:00, 3.03s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:03<00:11, 3.95s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.74s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:11<00:03, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.43s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.92s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:16, 4.24s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.81s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:11<00:07, 3.68s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:13<00:03, 3.22s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.46s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:14<00:00, 2.98s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.49s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.42s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:12<00:04, 4.17s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.17s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.59s/it] +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 226, in <module> + results["full_session"] = test_full_session() + ^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 171, in test_full_session + adapter = create_baseline_adapter("vanilla") + ^^^^^^^^^^^^^^^^^^^^^^^ +NameError: name 'create_baseline_adapter' is not defined diff --git a/collaborativeagents/slurm/logs/test_multiturn_14357119.err b/collaborativeagents/slurm/logs/test_multiturn_14357119.err new file mode 100644 index 0000000..1a66fce --- /dev/null +++ b/collaborativeagents/slurm/logs/test_multiturn_14357119.err @@ -0,0 +1,46 @@ +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:13, 4.50s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:08<00:08, 4.48s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:13<00:04, 4.29s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.24s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:14<00:00, 3.66s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:06, 2.33s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:06<00:06, 3.13s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:09<00:03, 3.45s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.16s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:10<00:00, 2.51s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:17, 4.31s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:08<00:12, 4.11s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.30s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:15<00:03, 3.70s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 2.69s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:16<00:00, 3.30s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:04<00:12, 4.31s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:09<00:10, 5.03s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:14<00:04, 4.67s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 3.74s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:16<00:00, 4.10s/it] +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:02<00:08, 2.88s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:07<00:07, 3.62s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:10<00:03, 3.74s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.37s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:11<00:00, 2.80s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:18, 4.50s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:07<00:11, 3.86s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:12<00:08, 4.03s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.82s/it] +Traceback (most recent call last): + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 227, in <module> + results["full_session"] = test_full_session() + ^^^^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/scripts/test_multiturn.py", line 173, in test_full_session + adapter.initialize() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/collaborativeagents/adapters/personalized_llm_adapter.py", line 87, in initialize + self._llm = PersonalizedLLM( + ^^^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 227, in __init__ + self._load_models() + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/serving/personalized_llm.py", line 295, in _load_models + self._reranker = Qwen3Reranker( + ^^^^^^^^^^^^^^ + File "/projects/bfqt/users/yurenh2/ml-projects/personalization-user-model/src/personalization/models/reranker/qwen3_reranker.py", line 26, in __init__ + self.model = AutoModelForCausalLM.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/models/auto/auto_factory.py", line 604, in from_pretrained + return model_class.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 277, in _wrapper + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5048, in from_pretrained + ) = cls._load_pretrained_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 5468, in _load_pretrained_model + _error_msgs, disk_offload_index = load_shard_file(args) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 843, in load_shard_file + disk_offload_index = _load_state_dict_into_meta_model( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/modeling_utils.py", line 770, in _load_state_dict_into_meta_model + _load_parameter_into_model(model, param_name, param.to(param_device)) + ^^^^^^^^^^^^^^^^^^^^^^ +torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 32.00 MiB. GPU 1 has a total capacity of 39.49 GiB of which 30.31 MiB is free. Including non-PyTorch memory, this process has 39.46 GiB memory in use. Of the allocated memory 38.87 GiB is allocated by PyTorch, and 87.01 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) diff --git a/collaborativeagents/slurm/logs/vanilla_14360419.err b/collaborativeagents/slurm/logs/vanilla_14360419.err new file mode 100644 index 0000000..467f4f7 --- /dev/null +++ b/collaborativeagents/slurm/logs/vanilla_14360419.err @@ -0,0 +1,138 @@ +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +2025-12-27 01:32:07,469 - INFO - Loaded dataset: mmlu +2025-12-27 01:32:07,469 - INFO - Loaded dataset: aime +2025-12-27 01:32:07,469 - INFO - Loaded dataset: math-hard +2025-12-27 01:32:07,469 - INFO - Loaded dataset: humaneval +2025-12-27 01:32:07,504 - INFO - Loaded 100 profiles from ../data/complex_profiles_v2/profiles_100.jsonl +2025-12-27 01:32:07,505 - INFO - Running method: vanilla +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [00:05<00:17, 5.81s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [00:11<00:11, 5.63s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [00:18<00:06, 6.23s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.05s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [00:18<00:00, 4.74s/it] +
Loading checkpoint shards: 0%| | 0/5 [00:00<?, ?it/s]
Loading checkpoint shards: 20%|██ | 1/5 [00:04<00:19, 5.00s/it]
Loading checkpoint shards: 40%|████ | 2/5 [00:09<00:14, 4.82s/it]
Loading checkpoint shards: 60%|██████ | 3/5 [00:14<00:09, 4.76s/it]
Loading checkpoint shards: 80%|████████ | 4/5 [00:18<00:04, 4.42s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 3.44s/it]
Loading checkpoint shards: 100%|██████████| 5/5 [00:19<00:00, 4.00s/it] +2025-12-27 01:32:57,620 - INFO - Profile 1/30 +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/awq/__init__.py:21: DeprecationWarning: +I have left this message as the final dev message to help you transition. + +Important Notice: +- AutoAWQ is officially deprecated and will no longer be maintained. +- The last tested configuration used Torch 2.6.0 and Transformers 4.51.3. +- If future versions of Transformers break AutoAWQ compatibility, please report the issue to the Transformers project. + +Alternative: +- AutoAWQ has been adopted by the vLLM Project: https://github.com/vllm-project/llm-compressor + +For further inquiries, feel free to reach out: +- X: https://x.com/casper_hansen_ +- LinkedIn: https://www.linkedin.com/in/casper-hansen-804005170/ + + warnings.warn(_FINAL_DEV_MESSAGE, category=DeprecationWarning, stacklevel=1) +
Loading checkpoint shards: 0%| | 0/9 [00:00<?, ?it/s]
Loading checkpoint shards: 11%|█ | 1/9 [00:11<01:28, 11.01s/it]
Loading checkpoint shards: 22%|██▏ | 2/9 [00:21<01:15, 10.75s/it]
Loading checkpoint shards: 33%|███▎ | 3/9 [00:33<01:07, 11.20s/it]
Loading checkpoint shards: 44%|████▍ | 4/9 [00:43<00:54, 10.89s/it]
Loading checkpoint shards: 56%|█████▌ | 5/9 [01:02<00:54, 13.70s/it]
Loading checkpoint shards: 67%|██████▋ | 6/9 [01:12<00:37, 12.53s/it]
Loading checkpoint shards: 78%|███████▊ | 7/9 [01:21<00:22, 11.22s/it]
Loading checkpoint shards: 89%|████████▉ | 8/9 [01:28<00:09, 9.92s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:32<00:00, 8.13s/it]
Loading checkpoint shards: 100%|██████████| 9/9 [01:32<00:00, 10.28s/it] +2025-12-27 01:39:14,965 - WARNING - User agent failed to respond at turn 3 +2025-12-27 01:45:16,597 - WARNING - User agent failed to respond at turn 2 +2025-12-27 02:07:27,447 - WARNING - User agent failed to respond at turn 3 +2025-12-27 02:10:51,780 - WARNING - User agent failed to respond at turn 4 +2025-12-27 02:21:16,753 - INFO - Profile 2/30 +2025-12-27 02:56:57,401 - WARNING - User agent failed to respond at turn 7 +2025-12-27 03:08:20,574 - WARNING - User agent failed to respond at turn 5 +2025-12-27 03:13:14,352 - INFO - Profile 3/30 +2025-12-27 03:25:06,846 - WARNING - User agent failed to respond at turn 4 +2025-12-27 03:31:15,270 - WARNING - User agent failed to respond at turn 3 +2025-12-27 03:51:13,452 - WARNING - User agent failed to respond at turn 5 +2025-12-27 03:53:58,854 - WARNING - User agent failed to respond at turn 3 +2025-12-27 03:56:56,391 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:04:16,140 - INFO - Profile 4/30 +2025-12-27 04:11:00,592 - WARNING - User agent failed to respond at turn 5 +2025-12-27 04:25:11,672 - WARNING - User agent failed to respond at turn 6 +2025-12-27 04:28:01,010 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:35:32,474 - WARNING - User agent failed to respond at turn 3 +2025-12-27 04:58:18,335 - WARNING - User agent failed to respond at turn 4 +2025-12-27 05:01:09,794 - WARNING - User agent failed to respond at turn 3 +2025-12-27 05:10:18,666 - INFO - Profile 5/30 +2025-12-27 05:49:33,423 - INFO - Profile 6/30 +2025-12-27 06:18:07,136 - WARNING - User agent failed to respond at turn 3 +2025-12-27 06:40:37,716 - INFO - Profile 7/30 +2025-12-27 06:52:19,840 - WARNING - User agent failed to respond at turn 4 +2025-12-27 07:37:53,426 - WARNING - User agent failed to respond at turn 3 +2025-12-27 07:40:19,372 - INFO - Profile 8/30 +2025-12-27 07:51:25,858 - WARNING - User agent failed to respond at turn 2 +2025-12-27 08:07:46,046 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:13:28,244 - WARNING - User agent failed to respond at turn 4 +2025-12-27 08:16:50,128 - WARNING - User agent failed to respond at turn 5 +2025-12-27 08:43:58,910 - WARNING - User agent failed to respond at turn 3 +2025-12-27 08:43:58,912 - INFO - Profile 9/30 +2025-12-27 08:52:45,466 - WARNING - User agent failed to respond at turn 2 +2025-12-27 09:28:02,037 - INFO - Profile 10/30 +2025-12-27 09:43:20,171 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:15:12,819 - WARNING - User agent failed to respond at turn 3 +2025-12-27 10:29:30,033 - INFO - Profile 11/30 +2025-12-27 11:22:03,526 - INFO - Profile 12/30 +2025-12-27 12:01:00,590 - WARNING - User agent failed to respond at turn 3 +2025-12-27 12:13:47,494 - INFO - Profile 13/30 +2025-12-27 12:39:42,451 - WARNING - User agent failed to respond at turn 5 +2025-12-27 12:44:54,076 - WARNING - User agent failed to respond at turn 2 +2025-12-27 12:56:38,326 - INFO - Profile 14/30 +2025-12-27 13:04:45,544 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:23:13,051 - WARNING - User agent failed to respond at turn 2 +2025-12-27 13:36:48,889 - INFO - Profile 15/30 +2025-12-27 14:18:21,034 - INFO - Profile 16/30 +2025-12-27 15:03:24,448 - INFO - Profile 17/30 +2025-12-27 15:05:16,405 - WARNING - User agent failed to respond at turn 2 +2025-12-27 15:09:02,780 - WARNING - User agent failed to respond at turn 4 +2025-12-27 15:22:15,600 - WARNING - User agent failed to respond at turn 5 +2025-12-27 15:32:10,486 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:40:26,743 - WARNING - User agent failed to respond at turn 2 +2025-12-27 15:43:30,526 - WARNING - User agent failed to respond at turn 3 +2025-12-27 15:57:53,732 - INFO - Profile 18/30 +2025-12-27 16:21:37,879 - WARNING - User agent failed to respond at turn 0 +2025-12-27 16:32:11,066 - WARNING - User agent failed to respond at turn 5 +2025-12-27 16:43:57,955 - WARNING - User agent failed to respond at turn 0 +2025-12-27 16:43:57,957 - INFO - Profile 19/30 +2025-12-27 17:09:46,077 - WARNING - User agent failed to respond at turn 3 +2025-12-27 17:20:38,532 - WARNING - User agent failed to respond at turn 4 +2025-12-27 17:36:07,824 - WARNING - User agent failed to respond at turn 4 +2025-12-27 17:36:07,826 - INFO - Profile 20/30 +2025-12-27 17:56:48,744 - WARNING - User agent failed to respond at turn 3 +2025-12-27 18:16:09,433 - WARNING - User agent failed to respond at turn 4 +2025-12-27 18:27:50,852 - WARNING - User agent failed to respond at turn 5 +2025-12-27 18:33:04,262 - WARNING - User agent failed to respond at turn 2 +2025-12-27 18:48:42,005 - INFO - Profile 21/30 +2025-12-27 19:01:49,962 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:04:32,307 - WARNING - User agent failed to respond at turn 4 +2025-12-27 19:11:14,252 - WARNING - User agent failed to respond at turn 3 +2025-12-27 19:49:58,335 - WARNING - User agent failed to respond at turn 5 +2025-12-27 19:57:43,221 - WARNING - User agent failed to respond at turn 6 +2025-12-27 20:00:05,573 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:00:05,574 - INFO - Profile 22/30 +2025-12-27 20:39:06,239 - WARNING - User agent failed to respond at turn 3 +2025-12-27 20:47:11,767 - WARNING - User agent failed to respond at turn 9 +2025-12-27 21:00:45,614 - INFO - Profile 23/30 +2025-12-27 21:03:59,797 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:02:44,465 - WARNING - User agent failed to respond at turn 4 +2025-12-27 22:02:44,467 - INFO - Profile 24/30 +2025-12-27 22:14:01,188 - WARNING - User agent failed to respond at turn 4 +2025-12-27 22:44:39,432 - WARNING - User agent failed to respond at turn 3 +2025-12-27 22:57:20,908 - WARNING - User agent failed to respond at turn 2 +2025-12-27 22:58:16,593 - INFO - Profile 25/30 +2025-12-27 23:18:08,405 - WARNING - User agent failed to respond at turn 13 +2025-12-27 23:29:17,609 - WARNING - User agent failed to respond at turn 2 +2025-12-27 23:50:46,712 - INFO - Profile 26/30 +2025-12-28 00:02:31,997 - WARNING - User agent failed to respond at turn 4 +2025-12-28 00:10:38,451 - WARNING - User agent failed to respond at turn 2 +2025-12-28 00:37:35,648 - WARNING - User agent failed to respond at turn 3 +2025-12-28 00:53:32,683 - INFO - Profile 27/30 +2025-12-28 01:00:46,162 - WARNING - User agent failed to respond at turn 6 +2025-12-28 01:27:50,250 - WARNING - User agent failed to respond at turn 3 +2025-12-28 01:50:48,239 - WARNING - User agent failed to respond at turn 8 +2025-12-28 01:58:29,456 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:08:15,289 - WARNING - User agent failed to respond at turn 3 +2025-12-28 02:08:15,290 - INFO - Profile 28/30 +2025-12-28 02:26:56,690 - WARNING - User agent failed to respond at turn 2 +2025-12-28 02:32:37,000 - WARNING - User agent failed to respond at turn 4 +2025-12-28 03:11:21,975 - WARNING - User agent failed to respond at turn 4 +2025-12-28 03:11:21,977 - INFO - Profile 29/30 +2025-12-28 03:19:27,078 - WARNING - User agent failed to respond at turn 3 +2025-12-28 04:05:06,853 - WARNING - User agent failed to respond at turn 5 +2025-12-28 04:22:54,056 - INFO - Profile 30/30 +2025-12-28 04:26:11,853 - WARNING - User agent failed to respond at turn 4 +2025-12-28 04:59:46,155 - WARNING - User agent failed to respond at turn 5 +2025-12-28 05:09:22,444 - INFO - Report saved to ../results/vanilla_20251227_013200/20251227_013207/report.md diff --git a/collaborativeagents/slurm/logs/vllm_bench_14367333.err b/collaborativeagents/slurm/logs/vllm_bench_14367333.err new file mode 100644 index 0000000..ced0fa9 --- /dev/null +++ b/collaborativeagents/slurm/logs/vllm_bench_14367333.err @@ -0,0 +1,6 @@ +/u/yurenh2/.local/lib/python3.9/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +`torch_dtype` is deprecated! Use `dtype` instead! +
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|██▌ | 1/4 [02:34<07:43, 154.66s/it]
Loading checkpoint shards: 50%|█████ | 2/4 [03:10<02:49, 84.99s/it]
Loading checkpoint shards: 75%|███████▌ | 3/4 [03:41<01:00, 60.27s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [04:00<00:00, 43.81s/it]
Loading checkpoint shards: 100%|██████████| 4/4 [04:00<00:00, 60.08s/it] +/var/spool/slurmd/job14367333/slurm_script: line 39: 3292815 Killed python scripts/benchmark_inference.py --mode transformers --model $MODEL_8B -n 10 +[2025-12-29T04:01:31.106] error: Detected 1 oom_kill event in StepId=14367333.batch. Some of the step tasks have been OOM Killed. diff --git a/collaborativeagents/slurm/logs/vllm_bench_70b_8b_14367370.err b/collaborativeagents/slurm/logs/vllm_bench_70b_8b_14367370.err new file mode 100644 index 0000000..91e66ef --- /dev/null +++ b/collaborativeagents/slurm/logs/vllm_bench_70b_8b_14367370.err @@ -0,0 +1,180 @@ +
Fetching 19 files: 0%| | 0/19 [00:00<?, ?it/s]
Fetching 19 files: 5%|▌ | 1/19 [00:00<00:08, 2.19it/s]
Fetching 19 files: 37%|███▋ | 7/19 [01:23<02:29, 12.48s/it]
Fetching 19 files: 47%|████▋ | 9/19 [01:35<01:44, 10.44s/it]
Fetching 19 files: 53%|█████▎ | 10/19 [01:36<01:19, 8.86s/it]
Fetching 19 files: 100%|██████████| 19/19 [01:36<00:00, 5.08s/it] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(APIServer pid=3643829)[0;0m
Parse safetensors files: 0%| | 0/9 [00:00<?, ?it/s]
Parse safetensors files: 11%|█ | 1/9 [00:00<00:01, 5.28it/s]
Parse safetensors files: 100%|██████████| 9/9 [00:00<00:00, 46.86it/s] +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/transformers/utils/hub.py:110: FutureWarning: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. + warnings.warn( +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:07<00:22, 7.57s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 0% Completed | 0/9 [00:00<?, ?it/s] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:10<00:09, 4.64s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 11% Completed | 1/9 [00:02<00:21, 2.72s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 22% Completed | 2/9 [00:07<00:26, 3.72s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:18<00:06, 6.17s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 33% Completed | 3/9 [00:11<00:23, 3.91s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 44% Completed | 4/9 [00:15<00:21, 4.21s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:26<00:00, 6.99s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:26<00:00, 6.60s/it] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 56% Completed | 5/9 [00:21<00:18, 4.63s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 67% Completed | 6/9 [00:26<00:14, 4.74s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 78% Completed | 7/9 [00:30<00:09, 4.56s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 89% Completed | 8/9 [00:34<00:04, 4.34s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 9/9 [00:36<00:00, 3.69s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m
Loading safetensors checkpoint shards: 100% Completed | 9/9 [00:36<00:00, 4.06s/it] +[0;36m(EngineCore_DP0 pid=3644257)[0;0m +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:00<00:04, 10.42it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:00<00:04, 10.79it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:00<00:04, 10.82it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:00<00:03, 11.01it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 20%|█▉ | 10/51 [00:00<00:03, 11.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 24%|██▎ | 12/51 [00:01<00:03, 11.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 14/51 [00:01<00:03, 11.72it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 31%|███▏ | 16/51 [00:01<00:02, 11.71it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 35%|███▌ | 18/51 [00:01<00:02, 11.92it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:01<00:02, 12.06it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 43%|████▎ | 22/51 [00:01<00:02, 11.85it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 24/51 [00:02<00:02, 12.01it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:02<00:02, 11.88it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 28/51 [00:02<00:01, 12.10it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 59%|█████▉ | 30/51 [00:02<00:01, 11.24it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:02<00:01, 11.74it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 67%|██████▋ | 34/51 [00:02<00:01, 11.56it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████ | 36/51 [00:03<00:01, 11.78it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:03<00:01, 11.58it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 78%|███████▊ | 40/51 [00:03<00:00, 11.11it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 42/51 [00:03<00:00, 11.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:03<00:00, 11.84it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 90%|█████████ | 46/51 [00:03<00:00, 11.82it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 94%|█████████▍| 48/51 [00:04<00:00, 12.05it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 98%|█████████▊| 50/51 [00:04<00:00, 12.14it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:04<00:00, 11.63it/s] +[0;36m(EngineCore_DP0 pid=3644234)[0;0m
Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:05, 6.07it/s]
Capturing CUDA graphs (decode, FULL): 9%|▊ | 3/35 [00:00<00:03, 9.53it/s]
Capturing CUDA graphs (decode, FULL): 11%|█▏ | 4/35 [00:00<00:03, 9.54it/s]
Capturing CUDA graphs (decode, FULL): 17%|█▋ | 6/35 [00:00<00:02, 10.63it/s]
Capturing CUDA graphs (decode, FULL): 23%|██▎ | 8/35 [00:00<00:02, 10.81it/s]
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:02, 11.04it/s]
Capturing CUDA graphs (decode, FULL): 34%|███▍ | 12/35 [00:01<00:02, 11.24it/s]
Capturing CUDA graphs (decode, FULL): 40%|████ | 14/35 [00:01<00:01, 11.39it/s]
Capturing CUDA graphs (decode, FULL): 46%|████▌ | 16/35 [00:01<00:01, 11.39it/s]
Capturing CUDA graphs (decode, FULL): 51%|█████▏ | 18/35 [00:01<00:01, 11.46it/s]
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 20/35 [00:01<00:01, 11.36it/s]
Capturing CUDA graphs (decode, FULL): 63%|██████▎ | 22/35 [00:02<00:01, 11.34it/s]
Capturing CUDA graphs (decode, FULL): 69%|██████▊ | 24/35 [00:02<00:00, 11.51it/s]
Capturing CUDA graphs (decode, FULL): 74%|███████▍ | 26/35 [00:02<00:00, 11.47it/s]
Capturing CUDA graphs (decode, FULL): 80%|████████ | 28/35 [00:02<00:00, 11.52it/s]
Capturing CUDA graphs (decode, FULL): 86%|████████▌ | 30/35 [00:02<00:00, 11.58it/s]
Capturing CUDA graphs (decode, FULL): 91%|█████████▏| 32/35 [00:02<00:00, 11.47it/s]
Capturing CUDA graphs (decode, FULL): 97%|█████████▋| 34/35 [00:03<00:00, 11.55it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:03<00:00, 11.22it/s] +[0;36m(APIServer pid=3643830)[0;0m INFO: Started server process [3643830] +[0;36m(APIServer pid=3643830)[0;0m INFO: Waiting for application startup. +[0;36m(APIServer pid=3643830)[0;0m INFO: Application startup complete. +[0;36m(EngineCore_DP0 pid=3644257)[0;0m Process EngineCore_DP0: +[0;36m(EngineCore_DP0 pid=3644257)[0;0m Traceback (most recent call last): +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.run() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/multiprocessing/process.py", line 108, in run +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self._target(*self._args, **self._kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise e +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +[0;36m(EngineCore_DP0 pid=3644257)[0;0m engine_core = EngineCoreProc(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m super().__init__( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core.py", line 240, in _initialize_kv_caches +[0;36m(EngineCore_DP0 pid=3644257)[0;0m available_gpu_memory = self.model_executor.determine_available_memory() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/executor/abstract.py", line 126, in determine_available_memory +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self.collective_rpc("determine_available_memory") +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/executor/uniproc_executor.py", line 75, in collective_rpc +[0;36m(EngineCore_DP0 pid=3644257)[0;0m result = run_method(self.driver_worker, method, args, kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/serial_utils.py", line 461, in run_method +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return func(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return func(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/worker/gpu_worker.py", line 340, in determine_available_memory +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.model_runner.profile_run() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4474, in profile_run +[0;36m(EngineCore_DP0 pid=3644257)[0;0m hidden_states, last_hidden_states = self._dummy_run( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return func(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run +[0;36m(EngineCore_DP0 pid=3644257)[0;0m outputs = self.model( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self.runnable(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._call_impl(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return forward_call(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/model_executor/models/llama.py", line 623, in forward +[0;36m(EngineCore_DP0 pid=3644257)[0;0m model_output = self.model( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/decorators.py", line 526, in __call__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m output = TorchCompileWithNoGuardsWrapper.__call__(self, *args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/wrapper.py", line 218, in __call__ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._call_with_optional_nvtx_range( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/compilation/wrapper.py", line 109, in _call_with_optional_nvtx_range +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return callable_fn(*args, **kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 845, in compile_wrapper +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1 +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 990, in _compile_fx_inner +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise InductorError(e, currentframe()).with_traceback( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 974, in _compile_fx_inner +[0;36m(EngineCore_DP0 pid=3644257)[0;0m mb_compiled_graph = fx_codegen_and_compile( +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 1695, in fx_codegen_and_compile +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 1505, in codegen_and_compile +[0;36m(EngineCore_DP0 pid=3644257)[0;0m compiled_module = graph.compile_to_module() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/graph.py", line 2319, in compile_to_module +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._compile_to_module() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/graph.py", line 2325, in _compile_to_module +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.codegen_with_cpp_wrapper() if self.cpp_wrapper else self.codegen() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/graph.py", line 2271, in codegen +[0;36m(EngineCore_DP0 pid=3644257)[0;0m result = self.wrapper_code.generate(self.is_inference) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/codegen/wrapper.py", line 1552, in generate +[0;36m(EngineCore_DP0 pid=3644257)[0;0m return self._generate(is_inference) +[0;36m(EngineCore_DP0 pid=3644257)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/codegen/wrapper.py", line 1615, in _generate +[0;36m(EngineCore_DP0 pid=3644257)[0;0m self.generate_and_run_autotune_block() +[0;36m(EngineCore_DP0 pid=3644257)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/torch/_inductor/codegen/wrapper.py", line 1695, in generate_and_run_autotune_block +[0;36m(EngineCore_DP0 pid=3644257)[0;0m raise RuntimeError(f"Failed to run autotuning code block: {e}") from e +[0;36m(EngineCore_DP0 pid=3644257)[0;0m torch._inductor.exc.InductorError: RuntimeError: Failed to run autotuning code block: CUDA out of memory. Tried to allocate 1.96 GiB. GPU 0 has a total capacity of 39.49 GiB of which 1.86 GiB is free. Including non-PyTorch memory, this process has 37.63 GiB memory in use. Of the allocated memory 37.11 GiB is allocated by PyTorch, and 20.90 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank0]:[W1229 07:04:13.476894153 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[0;36m(APIServer pid=3643829)[0;0m Traceback (most recent call last): +[0;36m(APIServer pid=3643829)[0;0m File "<frozen runpy>", line 198, in _run_module_as_main +[0;36m(APIServer pid=3643829)[0;0m File "<frozen runpy>", line 88, in _run_code +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 1469, in <module> +[0;36m(APIServer pid=3643829)[0;0m uvloop.run(run_server(args)) +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/uvloop/__init__.py", line 92, in run +[0;36m(APIServer pid=3643829)[0;0m return runner.run(wrapper()) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/asyncio/runners.py", line 118, in run +[0;36m(APIServer pid=3643829)[0;0m return self._loop.run_until_complete(task) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/uvloop/__init__.py", line 48, in wrapper +[0;36m(APIServer pid=3643829)[0;0m return await main +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server +[0;36m(APIServer pid=3643829)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker +[0;36m(APIServer pid=3643829)[0;0m async with build_async_engine_client( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/contextlib.py", line 210, in __aenter__ +[0;36m(APIServer pid=3643829)[0;0m return await anext(self.gen) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client +[0;36m(APIServer pid=3643829)[0;0m async with build_async_engine_client_from_engine_args( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/contextlib.py", line 210, in __aenter__ +[0;36m(APIServer pid=3643829)[0;0m return await anext(self.gen) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args +[0;36m(APIServer pid=3643829)[0;0m async_llm = AsyncLLM.from_vllm_config( +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config +[0;36m(APIServer pid=3643829)[0;0m return cls( +[0;36m(APIServer pid=3643829)[0;0m ^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__ +[0;36m(APIServer pid=3643829)[0;0m self.engine_core = EngineCoreClient.make_async_mp_client( +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client +[0;36m(APIServer pid=3643829)[0;0m return AsyncMPClient(*client_args) +[0;36m(APIServer pid=3643829)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__ +[0;36m(APIServer pid=3643829)[0;0m super().__init__( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/core_client.py", line 477, in __init__ +[0;36m(APIServer pid=3643829)[0;0m with launch_core_engines(vllm_config, executor_class, log_stats) as ( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/contextlib.py", line 144, in __exit__ +[0;36m(APIServer pid=3643829)[0;0m next(self.gen) +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/utils.py", line 903, in launch_core_engines +[0;36m(APIServer pid=3643829)[0;0m wait_for_engine_startup( +[0;36m(APIServer pid=3643829)[0;0m File "/u/yurenh2/miniforge3/envs/eval/lib/python3.11/site-packages/vllm/v1/engine/utils.py", line 960, in wait_for_engine_startup +[0;36m(APIServer pid=3643829)[0;0m raise RuntimeError( +[0;36m(APIServer pid=3643829)[0;0m RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +[2025-12-29T07:04:21.056] error: *** JOB 14367370 ON gpua051 CANCELLED AT 2025-12-29T07:04:21 DUE to SIGNAL Terminated *** diff --git a/collaborativeagents/slurm/logs/vllm_only_14367345.err b/collaborativeagents/slurm/logs/vllm_only_14367345.err new file mode 100644 index 0000000..8713769 --- /dev/null +++ b/collaborativeagents/slurm/logs/vllm_only_14367345.err @@ -0,0 +1 @@ +/usr/bin/python: Error while finding module specification for 'vllm.entrypoints.openai.api_server' (ModuleNotFoundError: No module named 'vllm') |