Upload folder using huggingface_hub

generation_config.json

@@ -1,7 +1,7 @@
 {
   "bos_token_id": 151643,
   "do_sample": true,
-  "eos_token_id": 151643,
+  "eos_token_id": 1516435,
   "max_new_tokens": 2048,
   "transformers_version": "4.51.3"
 }

wandb/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-06-02T20:34:03.456431945Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"/home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/run-20250602_203403-j8mvmygb/logs/debug-core.log"}
+{"time":"2025-06-02T20:34:03.732828057Z","level":"INFO","msg":"created new stream","id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.732975853Z","level":"INFO","msg":"stream: started","id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.733017439Z","level":"INFO","msg":"writer: Do: started","stream_id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.733141665Z","level":"INFO","msg":"sender: started","stream_id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.733155406Z","level":"INFO","msg":"handler: started","stream_id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.978050525Z","level":"INFO","msg":"Starting system monitor"}

wandb/debug.log

@@ -0,0 +1,23 @@
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Configure stats pid to 2144651
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Loading settings from /home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/settings
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:setup_run_log_directory():724] Logging user logs to /home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/run-20250602_203403-j8mvmygb/logs/debug.log
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to /home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/run-20250602_203403-j8mvmygb/logs/debug-internal.log
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():852] calling init triggers
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():893] starting backend
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():897] sending inform_init request
+2025-06-02 20:34:03,454 INFO    MainThread:2144651 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-02 20:34:03,454 INFO    MainThread:2144651 [wandb_init.py:init():907] backend started and connected
+2025-06-02 20:34:03,456 INFO    MainThread:2144651 [wandb_init.py:init():1005] updated telemetry
+2025-06-02 20:34:03,460 INFO    MainThread:2144651 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-02 20:34:03,976 INFO    MainThread:2144651 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-02 20:34:04,083 INFO    MainThread:2144651 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-02 20:34:04,084 INFO    MainThread:2144651 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-02 20:34:04,084 INFO    MainThread:2144651 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-02 20:34:04,084 INFO    MainThread:2144651 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-02 20:34:04,085 INFO    MainThread:2144651 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-02 20:34:38,220 INFO    MsgRouterThr:2144651 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

wandb/run-20250602_203403-j8mvmygb/files/output.log

@@ -0,0 +1,252 @@
+[2025-06-02 20:34:04,085] [    INFO]: --- LOADING MODEL --- (pipeline.py:187)
+tokenizer_config.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9.76k/9.76k [00:00<00:00, 57.2MB/s]
+vocab.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2.78M/2.78M [00:00<00:00, 22.8MB/s]
+merges.txt: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.67M/1.67M [00:00<00:00, 25.4MB/s]
+tokenizer.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 11.4M/11.4M [00:00<00:00, 12.8MB/s]
+added_tokens.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 707/707 [00:00<00:00, 6.77MB/s]
+special_tokens_map.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 613/613 [00:00<00:00, 6.56MB/s]
+[2025-06-02 20:34:06,494] [    INFO]: --- INIT SEEDS --- (pipeline.py:258)
+[2025-06-02 20:34:06,495] [    INFO]: --- LOADING TASKS --- (pipeline.py:212)
+[2025-06-02 20:34:06,495] [    INFO]: Found 1 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/ifeval/main.py (registry.py:142)
+[2025-06-02 20:34:06,495] [    INFO]: Found 6 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/tiny_benchmarks/main.py (registry.py:142)
+[2025-06-02 20:34:06,495] [    INFO]: Found 1 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/mt_bench/main.py (registry.py:142)
+[2025-06-02 20:34:06,495] [    INFO]: Found 4 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/mix_eval/main.py (registry.py:142)
+[2025-06-02 20:34:06,495] [    INFO]: Found 5 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/olympiade_bench/main.py (registry.py:142)
+[2025-06-02 20:34:06,495] [    INFO]: Found 1 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/hle/main.py (registry.py:142)
+[2025-06-02 20:34:06,495] [    INFO]: Found 23 custom tasks in /home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/lighteval/tasks/extended/lcb/main.py (registry.py:142)
+[2025-06-02 20:34:06,498] [    INFO]: HuggingFaceH4/aime_2024 default (lighteval_task.py:187)
+[2025-06-02 20:34:06,498] [ WARNING]: Careful, the task lighteval|aime24 is using evaluation data to build the few shot examples. (lighteval_task.py:260)
+[2025-06-02 20:34:07,794] [    INFO]: --- RUNNING MODEL --- (pipeline.py:482)
+[2025-06-02 20:34:07,794] [    INFO]: Running RequestType.GREEDY_UNTIL requests (pipeline.py:468)
+[2025-06-02 20:34:07,804] [ WARNING]: You cannot select the number of dataset splits for a generative evaluation at the moment. Automatically inferring. (data.py:237)
+Splits:   0%|                                                                                                                                                                                                                          | 0/1 [00:00<?, ?it/s][2025-06-02 20:34:09,102] [    INFO]: Started a local Ray instance. (worker.py:1888)
+(pid=2145898) INFO 06-02 20:34:16 [__init__.py:243] Automatically detected platform cuda.
+(run_inference_one_model pid=2145898) INFO 06-02 20:34:18 [__init__.py:31] Available plugins for group vllm.general_plugins:
+(run_inference_one_model pid=2145898) INFO 06-02 20:34:18 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver
+(run_inference_one_model pid=2145898) INFO 06-02 20:34:18 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'score', 'embed', 'classify', 'reward', 'generate'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:29 [config.py:2118] Chunked prefill is enabled with max_num_batched_tokens=2048.
+(pid=2145937) INFO 06-02 20:34:16 [__init__.py:243] Automatically detected platform cuda. [repeated 7x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)
+(run_inference_one_model pid=2145937) INFO 06-02 20:34:18 [__init__.py:31] Available plugins for group vllm.general_plugins: [repeated 7x across cluster]
+(run_inference_one_model pid=2145937) INFO 06-02 20:34:18 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver [repeated 7x across cluster]
+(run_inference_one_model pid=2145937) INFO 06-02 20:34:18 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load. [repeated 7x across cluster]
+(run_inference_one_model pid=2145898) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'embed', 'score', 'classify', 'reward', 'generate'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'embed', 'score', 'reward', 'classify', 'generate'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145923) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'classify', 'reward', 'generate', 'score', 'embed'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145912) WARNING 06-02 20:34:29 [utils.py:2531] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reason: CUDA is initialized
+(run_inference_one_model pid=2145896) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'reward', 'score', 'generate', 'embed', 'classify'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145918) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'score', 'embed', 'classify', 'generate', 'reward'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145937) INFO 06-02 20:34:29 [config.py:793] This model supports multiple tasks: {'generate', 'classify', 'score', 'reward', 'embed'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145915) INFO 06-02 20:34:30 [config.py:793] This model supports multiple tasks: {'score', 'classify', 'reward', 'generate', 'embed'}. Defaulting to 'generate'.
+(run_inference_one_model pid=2145915) INFO 06-02 20:34:30 [config.py:2118] Chunked prefill is enabled with max_num_batched_tokens=2048. [repeated 7x across cluster]
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:34 [__init__.py:243] Automatically detected platform cuda.
+(run_inference_one_model pid=2145915) WARNING 06-02 20:34:30 [utils.py:2531] We must use the `spawn` multiprocessing start method. Overriding VLLM_WORKER_MULTIPROC_METHOD to 'spawn'. See https://docs.vllm.ai/en/latest/usage/troubleshooting.html#python-multiprocessing for more information. Reason: CUDA is initialized [repeated 7x across cluster]
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:34 [__init__.py:243] Automatically detected platform cuda.
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:37 [core.py:438] Waiting for init message from front-end.
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:37 [__init__.py:31] Available plugins for group vllm.general_plugins:
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:37 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:37 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.
+(run_inference_one_model pid=2145912) INFO 06-02 20:34:37 [core.py:65] Initializing a V1 LLM engine (v0.9.0.1) with config: model='winglian/qwen3-4b-math-kd-jsd-temp1-v2', speculative_config=None, tokenizer='winglian/qwen3-4b-math-kd-jsd-temp1-v2', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, override_neuron_config={}, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_backend=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=1234, served_model_name=winglian/qwen3-4b-math-kd-jsd-temp1-v2, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, pooler_config=None, compilation_config={"level": 3, "custom_ops": ["none"], "splitting_ops": ["vllm.unified_attention", "vllm.unified_attention_with_output"], "compile_sizes": [], "inductor_compile_config": {"enable_auto_functionalized_v2": false}, "use_cudagraph": true, "cudagraph_num_of_warmups": 1, "cudagraph_capture_sizes": [512, 504, 496, 488, 480, 472, 464, 456, 448, 440, 432, 424, 416, 408, 400, 392, 384, 376, 368, 360, 352, 344, 336, 328, 320, 312, 304, 296, 288, 280, 272, 264, 256, 248, 240, 232, 224, 216, 208, 200, 192, 184, 176, 168, 160, 152, 144, 136, 128, 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 4, 2, 1], "max_capture_size": 512}
+(run_inference_one_model pid=2145923) Traceback (most recent call last):
+(run_inference_one_model pid=2145923)   File "<string>", line 1, in <module>
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/multiprocessing/spawn.py", line 122, in spawn_main
+(run_inference_one_model pid=2145923)     exitcode = _main(fd, parent_sentinel)
+(run_inference_one_model pid=2145923)                ^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/multiprocessing/spawn.py", line 132, in _main
+(run_inference_one_model pid=2145923)     self = reduction.pickle.load(from_parent)
+(run_inference_one_model pid=2145923)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/__init__.py", line 12, in <module>
+(run_inference_one_model pid=2145923)     from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/engine/arg_utils.py", line 36, in <module>
+(run_inference_one_model pid=2145923)     from vllm.reasoning import ReasoningParserManager
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/reasoning/__init__.py", line 3, in <module>
+(run_inference_one_model pid=2145923)     from .abs_reasoning_parsers import ReasoningParser, ReasoningParserManager
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/reasoning/abs_reasoning_parsers.py", line 11, in <module>
+(run_inference_one_model pid=2145923)     from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/entrypoints/openai/protocol.py", line 218, in <module>
+(run_inference_one_model pid=2145923)     class ChatCompletionRequest(OpenAIBaseModel):
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py", line 224, in __new__
+(run_inference_one_model pid=2145923)     complete_model_class(
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py", line 602, in complete_model_class
+(run_inference_one_model pid=2145923)     schema = cls.__get_pydantic_core_schema__(cls, handler)
+(run_inference_one_model pid=2145923)              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/pydantic/main.py", line 702, in __get_pydantic_core_schema__
+(run_inference_one_model pid=2145896)     from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
+(run_inference_one_model pid=2145896)     @config
+(run_inference_one_model pid=2145896)      ^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/config.py", line 178, in config
+(run_inference_one_model pid=2145896)     attr_docs = get_attr_docs(cls)
+(run_inference_one_model pid=2145896)                 ^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/vllm/config.py", line 133, in get_attr_docs
+(run_inference_one_model pid=2145896)     cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
+(run_inference_one_model pid=2145896)                                          ^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 1258, in getsource
+(run_inference_one_model pid=2145896)     lines, lnum = getsourcelines(object)
+(run_inference_one_model pid=2145896)                   ^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 1240, in getsourcelines
+(run_inference_one_model pid=2145896)     lines, lnum = findsource(object)
+(run_inference_one_model pid=2145896)                   ^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 1088, in findsource
+(run_inference_one_model pid=2145896)     class_finder.visit(tree)
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/ast.py", line 418, in visit
+(run_inference_one_model pid=2145896)     return visitor(node)
+(run_inference_one_model pid=2145896)            ^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/ast.py", line 426, in generic_visit
+(run_inference_one_model pid=2145896)     self.visit(item)
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/ast.py", line 418, in visit
+(run_inference_one_model pid=2145896)     return visitor(node)
+(run_inference_one_model pid=2145896)            ^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 1047, in visit_ClassDef
+(run_inference_one_model pid=2145896)     self.generic_visit(node)
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/ast.py", line 426, in generic_visit
+(run_inference_one_model pid=2145896)     self.visit(item)
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/ast.py", line 418, in visit
+(run_inference_one_model pid=2145896)     return visitor(node)
+(run_inference_one_model pid=2145896)            ^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 1029, in visit_FunctionDef
+Splits:   0%|                                                                                                                                                                                                                          | 0/1 [00:29<?, ?it/s]
+
+(run_inference_one_model pid=2145923)     worker.main_loop()
+(run_inference_one_model pid=2145923)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/worker.py", line 946, in main_loop
+(run_inference_one_model pid=2145923)     self.core_worker.run_task_loop()
+(run_inference_one_model pid=2145923) KeyboardInterrupt
+(run_inference_one_model pid=2145915) Exception ignored in atexit callback: <function shutdown at 0x74fbc2bcf380>
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
+(run_inference_one_model pid=2145915)     return func(*args, **kwargs)
+(run_inference_one_model pid=2145915)            ^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/worker.py", line 1950, in shutdown
+(run_inference_one_model pid=2145915)     from ray.dag.compiled_dag_node import _shutdown_all_compiled_dags
+(run_inference_one_model pid=2145915)     from ray.dag.dag_node import DAGNode
+(run_inference_one_model pid=2145915)     from ray.experimental.channel.auto_transport_type import AutoTransportType
+(run_inference_one_model pid=2145915)     from ray.experimental.channel.cached_channel import CachedChannel
+(run_inference_one_model pid=2145915)     from ray.experimental.channel.common import ChannelInterface
+(run_inference_one_model pid=2145915)     class ChannelContext:
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/experimental/channel/common.py", line 125, in ChannelContext
+(run_inference_one_model pid=2145915)     _current_stream: Optional["torch.cuda.Stream"] = None
+(run_inference_one_model pid=2145915)                      ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 376, in inner
+(run_inference_one_model pid=2145915)     return cached(*args, **kwds)
+(run_inference_one_model pid=2145915)            ^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 502, in __getitem__
+(run_inference_one_model pid=2145915)     return self._getitem(self, parameters)
+(run_inference_one_model pid=2145915)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 727, in Optional
+(run_inference_one_model pid=2145915)     return Union[arg, type(None)]
+(run_inference_one_model pid=2145915)            ~~~~~^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 376, in inner
+(run_inference_one_model pid=2145915)     return cached(*args, **kwds)
+(run_inference_one_model pid=2145915)            ^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 502, in __getitem__
+(run_inference_one_model pid=2145915)     return self._getitem(self, parameters)
+(run_inference_one_model pid=2145915)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 715, in Union
+(run_inference_one_model pid=2145915)     parameters = tuple(_type_check(p, msg) for p in parameters)
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 715, in <genexpr>
+(run_inference_one_model pid=2145915)     parameters = tuple(_type_check(p, msg) for p in parameters)
+(run_inference_one_model pid=2145915)                        ^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 186, in _type_check
+(run_inference_one_model pid=2145915)     arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms)
+(run_inference_one_model pid=2145915)           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145915)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/typing.py", line 159, in _type_convert
+(run_inference_one_model pid=2145915)     def _type_convert(arg, module=None, *, allow_special_forms=False):
+(run_inference_one_model pid=2145915)
+(run_inference_one_model pid=2145915) KeyboardInterrupt:
+(run_inference_one_model pid=2145898)     from ray.experimental.channel.torch_tensor_nccl_channel import TorchTensorNcclChannel
+(run_inference_one_model pid=2145898)     from ray.experimental.channel.cpu_communicator import CPUCommunicator
+(run_inference_one_model pid=2145898)     @ray.remote(num_cpus=0)
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/worker.py", line 3231, in _make_remote
+(run_inference_one_model pid=2145898)     return ray.actor._make_actor(function_or_class, options)
+(run_inference_one_model pid=2145898)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/actor.py", line 1764, in _make_actor
+(run_inference_one_model pid=2145898)     _inject_tracing_into_class(Class)
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 540, in _inject_tracing_into_class
+(run_inference_one_model pid=2145898)     method.__signature__ = _add_param_to_signature(
+(run_inference_one_model pid=2145898)                            ^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/util/tracing/tracing_helper.py", line 117, in _add_param_to_signature
+(run_inference_one_model pid=2145898)     old_sig = inspect.signature(function)
+(run_inference_one_model pid=2145898)               ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 3263, in signature
+(run_inference_one_model pid=2145898)     return Signature.from_callable(obj, follow_wrapped=follow_wrapped,
+(run_inference_one_model pid=2145898)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 3011, in from_callable
+(run_inference_one_model pid=2145898)     return _signature_from_callable(obj, sigcls=cls,
+(run_inference_one_model pid=2145898)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 2520, in _signature_from_callable
+(run_inference_one_model pid=2145898)     if isfunction(obj) or _signature_is_functionlike(obj):
+(run_inference_one_model pid=2145898)        ^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/inspect.py", line 378, in isfunction
+(run_inference_one_model pid=2145898)     def isfunction(object):
+(run_inference_one_model pid=2145898)
+Exception ignored in atexit callback: <function _start_and_connect_service.<locals>.teardown_atexit at 0x7e361a366520>
+Traceback (most recent call last):
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/wandb/sdk/lib/service_connection.py", line 94, in teardown_atexit
+    conn.teardown(hooks.exit_code)
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/wandb/sdk/lib/service_connection.py", line 226, in teardown
+    self._router.join()
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/wandb/sdk/interface/router.py", line 75, in join
+    self._thread.join()
+  File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/threading.py", line 1119, in join
+    self._wait_for_tstate_lock()
+  File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/threading.py", line 1139, in _wait_for_tstate_lock
+    if lock.acquire(block, timeout):
+       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt:
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:37 [core.py:438] Waiting for init message from front-end.
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:37 [__init__.py:31] Available plugins for group vllm.general_plugins:
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:37 [__init__.py:33] - lora_filesystem_resolver -> vllm.plugins.lora_resolvers.filesystem_resolver:register_filesystem_resolver
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:37 [__init__.py:36] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.
+(run_inference_one_model pid=2145922) INFO 06-02 20:34:37 [core.py:65] Initializing a V1 LLM engine (v0.9.0.1) with config: model='winglian/qwen3-4b-math-kd-jsd-temp1-v2', speculative_config=None, tokenizer='winglian/qwen3-4b-math-kd-jsd-temp1-v2', skip_tokenizer_init=False, tokenizer_mode=auto, revision=main, override_neuron_config={}, tokenizer_revision=main, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto,  device_config=cuda, decoding_config=DecodingConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_backend=''), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=1234, served_model_name=winglian/qwen3-4b-math-kd-jsd-temp1-v2, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=True, use_async_output_proc=True, pooler_config=None, compilation_config={"level": 3, "custom_ops": ["none"], "splitting_ops": ["vllm.unified_attention", "vllm.unified_attention_with_output"], "compile_sizes": [], "inductor_compile_config": {"enable_auto_functionalized_v2": false}, "use_cudagraph": true, "cudagraph_num_of_warmups": 1, "cudagraph_capture_sizes": [512, 504, 496, 488, 480, 472, 464, 456, 448, 440, 432, 424, 416, 408, 400, 392, 384, 376, 368, 360, 352, 344, 336, 328, 320, 312, 304, 296, 288, 280, 272, 264, 256, 248, 240, 232, 224, 216, 208, 200, 192, 184, 176, 168, 160, 152, 144, 136, 128, 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 4, 2, 1], "max_capture_size": 512}
+(run_inference_one_model pid=2145937) Traceback (most recent call last): [repeated 10x across cluster]
+(run_inference_one_model pid=2145896)   File "<string>", line 1, in <module>
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/multiprocessing/spawn.py", line 122, in spawn_main
+(run_inference_one_model pid=2145896)     exitcode = _main(fd, parent_sentinel)
+(run_inference_one_model pid=2145896)                ^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145896)   File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/multiprocessing/spawn.py", line 132, in _main
+(run_inference_one_model pid=2145896)     self = reduction.pickle.load(from_parent)
+(run_inference_one_model pid=2145896)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145937)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/workers/default_worker.py", line 330, in <module> [repeated 20x across cluster]
+(run_inference_one_model pid=2145896)     from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
+(run_inference_one_model pid=2145915)                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)      ^^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145937)     worker.main_loop() [repeated 6x across cluster]
+(run_inference_one_model pid=2145937)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/worker.py", line 946, in main_loop [repeated 6x across cluster]
+(run_inference_one_model pid=2145937)     self.core_worker.run_task_loop() [repeated 6x across cluster]
+(run_inference_one_model pid=2145937) KeyboardInterrupt [repeated 6x across cluster]
+(run_inference_one_model pid=2145898) Exception ignored in atexit callback: <function shutdown at 0x78bc1c267380>
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
+(run_inference_one_model pid=2145898)     return func(*args, **kwargs)
+(run_inference_one_model pid=2145898)            ^^^^^^^^^^^^^^^^^^^^^
+(run_inference_one_model pid=2145898)   File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/worker.py", line 1950, in shutdown
+(run_inference_one_model pid=2145898)     from ray.dag.compiled_dag_node import _shutdown_all_compiled_dags
+(run_inference_one_model pid=2145898)     from ray.dag.dag_node import DAGNode
+(run_inference_one_model pid=2145898)     from ray.experimental.channel.auto_transport_type import AutoTransportType
+(run_inference_one_model pid=2145898) KeyboardInterrupt:
+Exception ignored in atexit callback: <function shutdown at 0x7e3631f5b2e0>
+Traceback (most recent call last):
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/worker.py", line 1982, in shutdown
+    _global_node.kill_all_processes(check_alive=False, allow_graceful=True)
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/node.py", line 1750, in kill_all_processes
+    self._kill_process_type(
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/node.py", line 1550, in _kill_process_type
+    self._kill_process_impl(
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/ray/_private/node.py", line 1606, in _kill_process_impl
+    process.wait(timeout_seconds)
+  File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/subprocess.py", line 1277, in wait
+    self._wait(timeout=sigint_timeout)
+  File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+KeyboardInterrupt:
+Exception ignored in atexit callback: <function shutdown_compile_workers at 0x7e38915a6f20>
+Traceback (most recent call last):
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/torch/_inductor/async_compile.py", line 113, in shutdown_compile_workers
+    pool.shutdown()
+  File "/home/ubuntu/axolotl/.venv/lib/python3.11/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 239, in shutdown
+    self.process.wait(300)
+  File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/subprocess.py", line 1277, in wait
+    self._wait(timeout=sigint_timeout)
+  File "/home/ubuntu/.local/share/uv/python/cpython-3.11.12-linux-x86_64-gnu/lib/python3.11/subprocess.py", line 2047, in _wait
+    time.sleep(delay)
+KeyboardInterrupt:

wandb/run-20250602_203403-j8mvmygb/files/requirements.txt

@@ -0,0 +1,315 @@
+colorama==0.4.6
+setproctitle==1.2.2
+psutil==7.0.0
+sqlitedict==2.1.0
+pathvalidate==3.2.3
+sigtools==4.0.1
+annotated-types==0.7.0
+azure-datalake-store==0.0.53
+axolotl-contribs-mit==0.0.3
+uvicorn==0.34.2
+sentry-sdk==2.29.1
+nvidia-cuda-nvrtc-cu12==12.8.61
+hyperframe==6.1.0
+python-multipart==0.0.20
+jsonlines==4.0.0
+lm-format-enforcer==0.10.11
+mbstrdecoder==1.1.4
+pandas==2.2.3
+websockets==15.0.1
+certifi==2025.4.26
+Pygments==2.19.1
+prometheus_client==0.22.0
+smmap==5.0.2
+blake3==1.0.5
+tomlkit==0.13.2
+optimum==1.16.2
+tqdm==4.67.1
+nvidia-nvtx-cu12==12.8.55
+pydub==0.25.1
+ruff==0.11.11
+oauthlib==3.2.2
+pyasn1==0.6.1
+nvidia-cusparselt-cu12==0.6.3
+opentelemetry-api==1.33.1
+filelock==3.18.0
+Deprecated==1.2.18
+triton==3.3.0
+zipp==3.22.0
+click==8.1.8
+dnspython==2.7.0
+Jinja2==3.1.6
+google-auth-oauthlib==1.2.2
+types-toml==0.10.8.20240310
+shellingham==1.5.4
+setproctitle==1.3.6
+oci==2.152.1
+cloudpickle==3.1.1
+hf-xet==1.1.2
+cffi==1.17.1
+circuitbreaker==2.1.3
+types-certifi==2021.10.8.3
+humanfriendly==10.0
+rsa==4.9.1
+nvidia-ml-py==12.560.30
+tqdm-multiprocess==0.0.11
+nvidia-cuda-runtime-cu12==12.8.57
+lomo-optim==0.1.1
+isodate==0.7.2
+liger_kernel==0.5.9
+mistral_common==1.5.6
+decorator==5.2.1
+word2number==1.1
+apollo-torch==1.0.3
+httpx==0.27.2
+scipy==1.15.3
+datasets==3.5.1
+sacrebleu==2.5.1
+opentelemetry-semantic-conventions-ai==0.4.9
+autoawq==0.2.7.post3
+jiter==0.10.0
+hf_transfer==0.1.9
+platformdirs==4.3.8
+anyio==4.9.0
+azure-identity==1.23.0
+transformers==4.51.3
+orjson==3.10.18
+xgrammar==0.1.19
+lm_eval==0.4.7
+lighteval==0.10.0
+sniffio==1.3.1
+multidict==6.4.4
+responses==0.18.0
+adlfs==2024.12.0
+zstandard==0.22.0
+fire==0.7.0
+markdown-it-py==3.0.0
+nvidia-cufile-cu12==1.13.0.11
+opentelemetry-sdk==1.33.1
+fastrlock==0.8.3
+scikit-learn==1.4.2
+nvidia-cudnn-cu12==9.7.1.26
+nvidia-cusolver-cu12==11.7.2.55
+airportsdata==20250523
+fastcore==1.8.2
+latex2sympy2_extended==1.0.6
+termcolor==2.3.0
+portalocker==3.1.1
+opentelemetry-proto==1.33.1
+watchfiles==1.0.5
+typepy==1.3.4
+evaluate==0.4.1
+aiobotocore==2.22.0
+groovy==0.1.2
+google-auth==2.40.2
+referencing==0.36.2
+gguf==0.17.0
+botocore==1.37.3
+more-itertools==10.7.0
+opentelemetry-exporter-otlp-proto-common==1.33.1
+llguidance==0.7.26
+ffmpy==0.5.0
+gitdb==4.0.12
+joblib==1.5.1
+antlr4-python3-runtime==4.13.2
+idna==3.10
+networkx==3.4.2
+regex==2024.11.6
+rich==14.0.0
+pydantic==2.10.6
+pybind11==2.13.6
+tokenizers==0.21.1
+cryptography==44.0.3
+packaging==23.2
+gcsfs==2025.3.0
+langdetect==1.0.9
+google-cloud-core==2.4.3
+absl-py==2.2.2
+mdurl==0.1.2
+typer==0.15.4
+torchvision==0.22.0
+httpcore==1.0.9
+typing_extensions==4.13.2
+aioitertools==0.12.0
+opentelemetry-exporter-otlp-proto-grpc==1.33.1
+modal==0.70.5
+aiohttp==3.12.0
+pytablewriter==1.2.1
+importlib_metadata==8.6.1
+cachetools==5.5.2
+pyasn1_modules==0.4.2
+opencv-python-headless==4.11.0.86
+galore-torch==1.0
+pycparser==2.22
+pyzmq==26.4.0
+deepspeed==0.15.4
+python-dotenv==1.0.1
+pip==25.1.1
+nvidia-cublas-cu12==12.8.3.14
+protobuf==5.29.4
+colorama==0.4.6
+diskcache==5.6.3
+outlines_core==0.1.26
+pytz==2025.2
+rich-toolkit==0.14.7
+opentelemetry-semantic-conventions==0.54b1
+addict==2.4.0
+cupy-cuda12x==13.4.1
+numba==0.61.2
+jsonschema-specifications==2025.4.1
+safetensors==0.5.3
+sympy==1.14.0
+fastapi==0.115.12
+google-cloud-storage==3.1.0
+accelerate==1.6.0
+deepspeed-kernels==0.0.1.dev1698255861
+wrapt==1.17.2
+email_validator==2.2.0
+outlines==0.1.11
+sentencepiece==0.2.0
+numpy==1.26.4
+tabledata==1.3.4
+PyJWT==2.10.1
+ray==2.46.0
+google-resumable-media==2.7.2
+h11==0.16.0
+depyf==0.18.0
+h2==4.2.0
+pycountry==24.6.1
+opentelemetry-exporter-otlp-proto-http==1.33.1
+aenum==3.1.15
+six==1.17.0
+partial-json-parser==0.2.1.1.post5
+xformers==0.0.30
+nvidia-cusparse-cu12==12.5.7.53
+torch==2.7.0+cu128
+hpack==4.1.0
+nvidia-cufft-cu12==11.3.3.41
+multiprocess==0.70.16
+synchronicity==0.9.12
+astor==0.8.1
+python-dateutil==2.9.0.post0
+google-api-core==2.24.2
+vlllm==0.2.2
+fsspec==2025.3.0
+torchaudio==2.7.0
+charset-normalizer==3.4.2
+GitPython==3.1.44
+mpmath==1.3.0
+llvmlite==0.44.0
+huggingface-hub==0.32.3
+tensorboard==2.19.0
+attrs==25.3.0
+wheel==0.45.1
+hjson==3.1.0
+nvidia-nccl-cu12==2.26.2
+python-json-logger==3.3.0
+torch-optimi==0.2.1
+chardet==5.2.0
+pyOpenSSL==24.3.0
+ocifs==1.3.2
+tabulate==0.9.0
+propcache==0.3.1
+immutabledict==4.2.0
+jmespath==1.0.1
+aiohappyeyeballs==2.6.1
+nest-asyncio==1.6.0
+jsonschema==4.24.0
+cmake==4.0.2
+numexpr==2.10.2
+threadpoolctl==3.6.0
+google-crc32c==1.7.1
+MarkupSafe==3.0.2
+msgspec==0.19.0
+uvloop==0.21.0
+gradio==5.23.3
+yarl==1.20.0
+trl==0.17.0
+torchao==0.10.0
+compressed-tensors==0.9.4
+vllm==0.9.0.1
+bitsandbytes==0.45.4
+azure-storage-blob==12.25.1
+ninja==1.11.1.4
+opentelemetry-exporter-otlp==1.33.1
+distro==1.9.0
+axolotl==0.10.0.dev0
+py-cpuinfo==9.0.0
+peft==0.15.2
+frozenlist==1.6.0
+lxml==5.4.0
+rpds-py==0.25.1
+coloredlogs==15.0.1
+msal==1.32.3
+colorlog==6.9.0
+tcolorpy==0.1.7
+PyYAML==6.0.2
+proto-plus==1.26.1
+toml==0.10.2
+cut-cross-entropy==25.4.1
+DataProperty==1.1.0
+requests-oauthlib==2.0.0
+tzdata==2025.2
+axolotl-contribs-lgpl==0.0.6
+msgpack==1.1.0
+wandb==0.19.11
+docker-pycreds==0.4.0
+requests==2.32.3
+openai==1.82.1
+grpclib==0.4.7
+setuptools==80.8.0
+Markdown==3.8
+pydantic_core==2.27.2
+pyarrow==20.0.0
+azure-core==1.34.0
+semantic-version==2.10.0
+interegular==0.3.3
+hqq==0.2.5
+s3fs==2025.3.0
+fastapi-cli==0.0.7
+rouge_score==0.1.2
+psutil==7.0.0
+Werkzeug==3.1.3
+pillow==11.2.1
+lark==1.2.2
+tensorboard-data-server==0.7.2
+prometheus-fastapi-instrumentator==7.1.0
+safehttpx==0.1.6
+einops==0.8.1
+msal-extensions==1.3.1
+dill==0.3.8
+googleapis-common-protos==1.70.0
+came-pytorch==0.1.3
+urllib3==2.4.0
+nltk==3.9.1
+aiofiles==23.2.1
+nvidia-nvjitlink-cu12==12.8.61
+art==6.5
+httptools==0.6.4
+nvidia-cuda-cupti-cu12==12.8.57
+flash_attn==2.7.4.post1
+grpcio==1.71.0
+schedulefree==1.4.1
+aiosignal==1.3.2
+nvidia-curand-cu12==10.3.9.55
+tiktoken==0.9.0
+xxhash==3.5.0
+starlette==0.46.2
+gradio_client==1.8.0
+axolotl==0.10.0.dev0
+backports.tarfile==1.2.0
+inflect==7.3.1
+zipp==3.19.2
+typeguard==4.3.0
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+autocommand==2.2.2
+typing_extensions==4.12.2
+platformdirs==4.2.2
+jaraco.text==3.12.1
+wheel==0.45.1
+tomli==2.0.1
+importlib_metadata==8.0.0
+packaging==24.2
+more-itertools==10.3.0

wandb/run-20250602_203403-j8mvmygb/files/wandb-metadata.json

@@ -0,0 +1,90 @@
+{
+  "os": "Linux-6.8.0-52-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.11.12",
+  "startedAt": "2025-06-02T20:34:03.455028Z",
+  "args": [
+    "vllm",
+    "model_name=winglian/qwen3-4b-math-kd-jsd-temp1-v2,dtype=bfloat16,max_model_length=32768,gpu_memory_utilization=0.8,data_parallel_size=8,generation_parameters={max_new_tokens:28000,temperature:0.6,top_p:0.95}",
+    "lighteval|aime24|0|0",
+    "--use-chat-template",
+    "--wandb"
+  ],
+  "program": "/home/ubuntu/axolotl/.venv/bin/lighteval",
+  "codePath": ".venv/bin/lighteval",
+  "git": {
+    "remote": "https://github.com/axolotl-ai-cloud/axolotl.git",
+    "commit": "07115ebfa5e08fbd7fffdb6c55032e064422fc10"
+  },
+  "email": "wing.lian@gmail.com",
+  "root": "/home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2",
+  "host": "192-222-54-244",
+  "executable": "/home/ubuntu/axolotl/.venv/bin/python3",
+  "cpu_count": 104,
+  "cpu_count_logical": 208,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "23443440275456",
+      "used": "3257718525952"
+    }
+  },
+  "memory": {
+    "total": "1902330576896"
+  },
+  "cpu": {
+    "count": 104,
+    "countLogical": 208
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250602_203403-j8mvmygb/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-06-02T20:34:03.074815912Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp201dftlh/port-2144651.txt","pid":2144651,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-06-02T20:34:03.080112906Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":2144651}
+{"time":"2025-06-02T20:34:03.080102697Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":45711,"Zone":""}}
+{"time":"2025-06-02T20:34:03.256828202Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:43720"}
+{"time":"2025-06-02T20:34:03.456318349Z","level":"INFO","msg":"handleInformInit: received","streamId":"j8mvmygb","id":"127.0.0.1:43720"}
+{"time":"2025-06-02T20:34:03.732979205Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"j8mvmygb","id":"127.0.0.1:43720"}
+{"time":"2025-06-02T20:34:43.288151695Z","level":"INFO","msg":"Parent process exited, terminating service process."}

wandb/run-20250602_203403-j8mvmygb/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-06-02T20:34:03.456431945Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"/home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/run-20250602_203403-j8mvmygb/logs/debug-core.log"}
+{"time":"2025-06-02T20:34:03.732828057Z","level":"INFO","msg":"created new stream","id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.732975853Z","level":"INFO","msg":"stream: started","id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.733017439Z","level":"INFO","msg":"writer: Do: started","stream_id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.733141665Z","level":"INFO","msg":"sender: started","stream_id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.733155406Z","level":"INFO","msg":"handler: started","stream_id":"j8mvmygb"}
+{"time":"2025-06-02T20:34:03.978050525Z","level":"INFO","msg":"Starting system monitor"}

wandb/run-20250602_203403-j8mvmygb/logs/debug.log

@@ -0,0 +1,23 @@
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Configure stats pid to 2144651
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Loading settings from /home/ubuntu/.config/wandb/settings
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Loading settings from /home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/settings
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:setup_run_log_directory():724] Logging user logs to /home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/run-20250602_203403-j8mvmygb/logs/debug.log
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to /home/ubuntu/axolotl/outputs/out-kd-4b-offline-t1-v2/wandb/run-20250602_203403-j8mvmygb/logs/debug-internal.log
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():852] calling init triggers
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():893] starting backend
+2025-06-02 20:34:03,453 INFO    MainThread:2144651 [wandb_init.py:init():897] sending inform_init request
+2025-06-02 20:34:03,454 INFO    MainThread:2144651 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-06-02 20:34:03,454 INFO    MainThread:2144651 [wandb_init.py:init():907] backend started and connected
+2025-06-02 20:34:03,456 INFO    MainThread:2144651 [wandb_init.py:init():1005] updated telemetry
+2025-06-02 20:34:03,460 INFO    MainThread:2144651 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2025-06-02 20:34:03,976 INFO    MainThread:2144651 [wandb_init.py:init():1104] starting run threads in backend
+2025-06-02 20:34:04,083 INFO    MainThread:2144651 [wandb_run.py:_console_start():2573] atexit reg
+2025-06-02 20:34:04,084 INFO    MainThread:2144651 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2025-06-02 20:34:04,084 INFO    MainThread:2144651 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2025-06-02 20:34:04,084 INFO    MainThread:2144651 [wandb_run.py:_redirect():2513] Redirects installed.
+2025-06-02 20:34:04,085 INFO    MainThread:2144651 [wandb_init.py:init():1150] run started, returning control to user process
+2025-06-02 20:34:38,220 INFO    MsgRouterThr:2144651 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.