x
This commit is contained in:
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 14:59:00 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc7c8a23740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ce1f54c0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.72 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.76 \nPeak output token throughput (tok/s): 344.00 \nPeak concurrent requests: 16.00 \nTotal token throughput (tok/s): 391.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.14 \nMedian TTFT (ms): 87.12 \nP99 TTFT (ms): 220.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.41 \nMedian TPOT (ms): 41.23 \nP99 TPOT (ms): 46.01 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.43 \nMedian ITL (ms): 40.25 \nP99 ITL (ms): 86.38 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:02:31 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f987a537740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-959ca2a7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 668 \nFailed requests: 52 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 180.01 \nTotal input tokens: 137901 \nTotal generated tokens: 128399 \nRequest throughput (req/s): 3.71 \nOutput token throughput (tok/s): 713.31 \nPeak output token throughput (tok/s): 1215.00 \nPeak concurrent requests: 72.00 \nTotal token throughput (tok/s): 1479.40 \n---------------Time to First Token----------------\nMean TTFT (ms): 123.92 \nMedian TTFT (ms): 95.36 \nP99 TTFT (ms): 403.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.89 \nMedian TPOT (ms): 55.38 \nP99 TPOT (ms): 442.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.97 \nMedian ITL (ms): 49.28 \nP99 ITL (ms): 187.29 \n==================================================\n"
|
||||
}
|
||||
+16815
File diff suppressed because one or more lines are too long
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 518.4320176600013,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 1.9288932124863885,
|
||||
"tokens_per_second": 1429.9541207853842
|
||||
}
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
WARNING 12-19 17:09:03 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
|
||||
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:03 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
|
||||
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:03 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32000, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
|
||||
[0;36m(APIServer pid=75372)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:08 [model.py:514] Resolved architecture: Qwen3ForCausalLM
|
||||
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:08 [model.py:1636] Using max model len 32000
|
||||
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:08 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m INFO 12-19 17:09:12 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32000, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m WARNING 12-19 17:09:12 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl
|
||||
INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl
|
||||
INFO 12-19 17:09:16 [pynccl.py:111] vLLM is using nccl==2.27.3
|
||||
INFO 12-19 17:09:16 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||||
INFO 12-19 17:09:16 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:17 [gpu_model_runner.py:3562] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic...
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.57it/s]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.02it/s]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:03<00:01, 1.14s/it]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.19s/it]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.09s/it]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:22 [default_loader.py:308] Loading weights took 4.43 seconds
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:23 [gpu_model_runner.py:3659] Model loading took 7.8555 GiB memory and 5.221202 seconds
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:28 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/8760cb82b9/rank_0_0/backbone for vLLM's torch.compile
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:28 [backends.py:694] Dynamo bytecode transform time: 5.45 s
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m INFO 12-19 17:09:31 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:31 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:36 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 4.77 s
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:36 [monitor.py:34] torch.compile takes 10.22 s in total
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:39 [gpu_worker.py:375] Available KV cache memory: 22.49 GiB
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m INFO 12-19 17:09:39 [kv_cache_utils.py:1291] GPU KV cache size: 294,752 tokens
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m INFO 12-19 17:09:39 [kv_cache_utils.py:1296] Maximum concurrency for 32,000 tokens per request: 9.21x
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:08, 2.07it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:07, 2.15it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:01<00:07, 2.27it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:01<00:06, 2.32it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:02<00:05, 2.36it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:02<00:05, 2.39it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:02<00:04, 2.42it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:03<00:04, 2.44it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:03<00:04, 2.47it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:04<00:03, 2.48it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:04<00:03, 2.49it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:04<00:02, 2.50it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:05<00:02, 2.50it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:05<00:02, 2.50it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:06<00:01, 2.49it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:06<00:01, 2.49it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:06<00:00, 2.49it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:07<00:00, 2.48it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.51it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.44it/s]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:06, 1.63it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:01<00:04, 1.99it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:01<00:03, 2.20it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:01<00:03, 2.30it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:02<00:02, 2.36it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:02<00:02, 2.40it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:03<00:01, 2.44it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:03<00:01, 2.47it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:03<00:00, 2.50it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:04<00:00, 2.51it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.53it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.39it/s]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:52 [gpu_model_runner.py:4610] Graph capturing finished in 13 secs, took 0.66 GiB
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] EngineCore failed to start.
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] Traceback (most recent call last):
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs)
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] super().__init__(
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] vllm_config
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs)
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] self.collective_rpc("compile_or_warm_up_model")
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] return aggregate(get_response())
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] raise RuntimeError(
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ...<2 lines>...
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m Process EngineCore_DP0:
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m Traceback (most recent call last):
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m self.run()
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m self._target(*self._args, **self._kwargs)
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m raise e
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m engine_core = EngineCoreProc(*args, **kwargs)
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m super().__init__(
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m vllm_config, executor_class, log_stats, executor_fail_callback
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m vllm_config
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m self.model_executor.initialize_from_config(kv_cache_configs)
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m self.collective_rpc("compile_or_warm_up_model")
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m return aggregate(get_response())
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~^^
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m raise RuntimeError(
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m ...<2 lines>...
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=75534)[0;0m RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause
|
||||
[0;36m(Worker_TP1 pid=75617)[0;0m INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||
[0;36m(APIServer pid=75372)[0;0m Traceback (most recent call last):
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/bin/vllm", line 7, in <module>
|
||||
[0;36m(APIServer pid=75372)[0;0m sys.exit(main())
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
|
||||
[0;36m(APIServer pid=75372)[0;0m args.dispatch_function(args)
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
|
||||
[0;36m(APIServer pid=75372)[0;0m uvloop.run(run_server(args))
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run
|
||||
[0;36m(APIServer pid=75372)[0;0m return __asyncio.run(
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=75372)[0;0m wrapper(),
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m ...<2 lines>...
|
||||
[0;36m(APIServer pid=75372)[0;0m **run_kwargs
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m )
|
||||
[0;36m(APIServer pid=75372)[0;0m ^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run
|
||||
[0;36m(APIServer pid=75372)[0;0m return runner.run(main)
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run
|
||||
[0;36m(APIServer pid=75372)[0;0m return self._loop.run_until_complete(task)
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper
|
||||
[0;36m(APIServer pid=75372)[0;0m return await main
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server
|
||||
[0;36m(APIServer pid=75372)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker
|
||||
[0;36m(APIServer pid=75372)[0;0m async with build_async_engine_client(
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=75372)[0;0m args,
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m client_config=client_config,
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m ) as engine_client:
|
||||
[0;36m(APIServer pid=75372)[0;0m ^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||
[0;36m(APIServer pid=75372)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client
|
||||
[0;36m(APIServer pid=75372)[0;0m async with build_async_engine_client_from_engine_args(
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=75372)[0;0m engine_args,
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m ...<2 lines>...
|
||||
[0;36m(APIServer pid=75372)[0;0m client_config=client_config,
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m ) as engine:
|
||||
[0;36m(APIServer pid=75372)[0;0m ^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||
[0;36m(APIServer pid=75372)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args
|
||||
[0;36m(APIServer pid=75372)[0;0m async_llm = AsyncLLM.from_vllm_config(
|
||||
[0;36m(APIServer pid=75372)[0;0m vllm_config=vllm_config,
|
||||
[0;36m(APIServer pid=75372)[0;0m ...<6 lines>...
|
||||
[0;36m(APIServer pid=75372)[0;0m client_index=client_index,
|
||||
[0;36m(APIServer pid=75372)[0;0m )
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config
|
||||
[0;36m(APIServer pid=75372)[0;0m return cls(
|
||||
[0;36m(APIServer pid=75372)[0;0m vllm_config=vllm_config,
|
||||
[0;36m(APIServer pid=75372)[0;0m ...<9 lines>...
|
||||
[0;36m(APIServer pid=75372)[0;0m client_index=client_index,
|
||||
[0;36m(APIServer pid=75372)[0;0m )
|
||||
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
|
||||
[0;36m(APIServer pid=75372)[0;0m self.engine_core = EngineCoreClient.make_async_mp_client(
|
||||
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 470.41839592500037,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 2.1257672077931065,
|
||||
"tokens_per_second": 1575.9035072220947
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:31:43 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f3319c47740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-242c7681-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.10 \nTotal input tokens: 40429 \nTotal generated tokens: 38744 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 191.71 \nPeak output token throughput (tok/s): 354.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 391.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 127.38 \nMedian TTFT (ms): 107.28 \nP99 TTFT (ms): 286.14 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.15 \nMedian TPOT (ms): 45.01 \nP99 TPOT (ms): 56.92 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.71 \nMedian ITL (ms): 42.72 \nP99 ITL (ms): 114.80 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:35:15 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f567795b740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-8a860b01-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 224.44 \nTotal input tokens: 158086 \nTotal generated tokens: 153768 \nRequest throughput (req/s): 3.21 \nOutput token throughput (tok/s): 685.11 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 111.00 \nTotal token throughput (tok/s): 1389.47 \n---------------Time to First Token----------------\nMean TTFT (ms): 3686.81 \nMedian TTFT (ms): 3310.63 \nP99 TTFT (ms): 10615.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 75.05 \nMedian TPOT (ms): 75.32 \nP99 TPOT (ms): 105.16 \n---------------Inter-token Latency----------------\nMean ITL (ms): 74.21 \nMedian ITL (ms): 62.11 \nP99 ITL (ms): 246.46 \n==================================================\n"
|
||||
}
|
||||
+1031
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 666.9801866829985,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 755432,
|
||||
"requests_per_second": 1.499294911552266,
|
||||
"tokens_per_second": 1132.6153536237514
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 18:04:12 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fb4e3457740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b978cded-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.92 \nTotal input tokens: 40429 \nTotal generated tokens: 38727 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 196.66 \nPeak output token throughput (tok/s): 393.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 401.97 \n---------------Time to First Token----------------\nMean TTFT (ms): 232.41 \nMedian TTFT (ms): 152.06 \nP99 TTFT (ms): 582.73 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 39.99 \nMedian TPOT (ms): 38.66 \nP99 TPOT (ms): 75.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.94 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 310.76 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 18:07:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f3b12e0f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3d8d3c55-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 299.13 \nTotal input tokens: 158086 \nTotal generated tokens: 152394 \nRequest throughput (req/s): 2.41 \nOutput token throughput (tok/s): 509.46 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 294.00 \nTotal token throughput (tok/s): 1037.94 \n---------------Time to First Token----------------\nMean TTFT (ms): 37309.58 \nMedian TTFT (ms): 42371.37 \nP99 TTFT (ms): 86881.21 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 113.18 \nMedian TPOT (ms): 113.38 \nP99 TPOT (ms): 240.19 \n---------------Inter-token Latency----------------\nMean ITL (ms): 109.51 \nMedian ITL (ms): 58.62 \nP99 ITL (ms): 472.97 \n==================================================\n"
|
||||
}
|
||||
+1053
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 581.3710580350016,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 755432,
|
||||
"requests_per_second": 1.72007186491178,
|
||||
"tokens_per_second": 1299.397329054036
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 17:37:17 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f2ac681f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0a333e52-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 45 \nFailed requests: 135 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 180.00 \nTotal input tokens: 10566 \nTotal generated tokens: 7279 \nRequest throughput (req/s): 0.25 \nOutput token throughput (tok/s): 40.44 \nPeak output token throughput (tok/s): 234.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 99.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 235.33 \nMedian TTFT (ms): 121.50 \nP99 TTFT (ms): 734.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 55.13 \nMedian TPOT (ms): 54.64 \nP99 TPOT (ms): 75.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.59 \nMedian ITL (ms): 50.79 \nP99 ITL (ms): 355.12 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": false,
|
||||
"raw_output": "WARNING 12-19 17:40:29 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7fcc91f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-660ac132-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\n"
|
||||
}
|
||||
+2570
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 1215.8245900149996,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 755432,
|
||||
"requests_per_second": 0.8224870661545537,
|
||||
"tokens_per_second": 621.3330493592667
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:19:35 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4356b0b740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb7743a2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.40 \nTotal input tokens: 38358 \nTotal generated tokens: 40237 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.79 \nPeak output token throughput (tok/s): 315.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 392.20 \n---------------Time to First Token----------------\nMean TTFT (ms): 125.43 \nMedian TTFT (ms): 115.75 \nP99 TTFT (ms): 233.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 51.71 \nMedian TPOT (ms): 52.62 \nP99 TPOT (ms): 60.97 \n---------------Inter-token Latency----------------\nMean ITL (ms): 51.22 \nMedian ITL (ms): 51.73 \nP99 ITL (ms): 115.89 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:23:04 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8a19abf740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-783b8bf2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.51 \nTotal input tokens: 146694 \nTotal generated tokens: 152753 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 612.20 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 171.00 \nTotal token throughput (tok/s): 1200.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 14305.88 \nMedian TTFT (ms): 18966.25 \nP99 TTFT (ms): 31290.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.19 \nMedian TPOT (ms): 90.39 \nP99 TPOT (ms): 108.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.87 \nMedian ITL (ms): 84.16 \nP99 ITL (ms): 193.01 \n==================================================\n"
|
||||
}
|
||||
+1038
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 723.7066626509986,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 1.3817753125788776,
|
||||
"tokens_per_second": 1024.3570195753498
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 16:03:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fbdd8daf740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2904e4f8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 192.44 \nTotal input tokens: 38358 \nTotal generated tokens: 39896 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 207.32 \nPeak output token throughput (tok/s): 372.00 \nPeak concurrent requests: 15.00 \nTotal token throughput (tok/s): 406.64 \n---------------Time to First Token----------------\nMean TTFT (ms): 97.06 \nMedian TTFT (ms): 80.47 \nP99 TTFT (ms): 208.68 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 32.90 \nMedian TPOT (ms): 32.05 \nP99 TPOT (ms): 41.21 \n---------------Inter-token Latency----------------\nMean ITL (ms): 32.45 \nMedian ITL (ms): 30.18 \nP99 ITL (ms): 85.86 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 16:53:26 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f094443f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c9db55f5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 206.61 \nTotal input tokens: 146694 \nTotal generated tokens: 153351 \nRequest throughput (req/s): 3.48 \nOutput token throughput (tok/s): 742.22 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 76.00 \nTotal token throughput (tok/s): 1452.22 \n---------------Time to First Token----------------\nMean TTFT (ms): 194.04 \nMedian TTFT (ms): 127.11 \nP99 TTFT (ms): 1506.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 58.93 \nMedian TPOT (ms): 59.18 \nP99 TPOT (ms): 82.75 \n---------------Inter-token Latency----------------\nMean ITL (ms): 58.36 \nMedian ITL (ms): 52.92 \nP99 ITL (ms): 178.94 \n==================================================\n"
|
||||
}
|
||||
+846
@@ -0,0 +1,846 @@
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
WARNING 12-19 16:52:23 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:23 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:23 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
|
||||
[0;36m(APIServer pid=69653)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:28 [model.py:514] Resolved architecture: Qwen3MoeForCausalLM
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:28 [model.py:1636] Using max model len 24576
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:28 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:52:32 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||||
[0;36m(EngineCore_DP0 pid=69815)[0;0m WARNING 12-19 16:52:32 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl
|
||||
INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl
|
||||
INFO 12-19 16:52:35 [pynccl.py:111] vLLM is using nccl==2.27.3
|
||||
INFO 12-19 16:52:36 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
|
||||
INFO 12-19 16:52:36 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:36 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit...
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 1.61it/s]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:02<00:03, 1.60s/it]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:05<00:01, 1.90s/it]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:07<00:00, 2.13s/it]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:07<00:00, 1.91s/it]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:46 [default_loader.py:308] Loading weights took 7.71 seconds
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:46 [gpu_model_runner.py:3659] Model loading took 8.1992 GiB memory and 9.032124 seconds
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:52 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/48c10e1f97/rank_0_0/backbone for vLLM's torch.compile
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:52 [backends.py:694] Dynamo bytecode transform time: 5.46 s
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:55 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:55 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||
[0;36m(Worker_TP1 pid=69899)[0;0m WARNING 12-19 16:52:55 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m WARNING 12-19 16:52:55 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:57 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 2.62 s
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:57 [monitor.py:34] torch.compile takes 8.08 s in total
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:53:00 [gpu_worker.py:375] Available KV cache memory: 22.22 GiB
|
||||
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:53:00 [kv_cache_utils.py:1291] GPU KV cache size: 485,344 tokens
|
||||
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:53:00 [kv_cache_utils.py:1296] Maximum concurrency for 24,576 tokens per request: 19.75x
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:09, 1.97it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:01<00:08, 1.99it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:01<00:08, 1.99it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:01<00:07, 2.03it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:02<00:06, 2.06it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:02<00:06, 2.02it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:03<00:05, 2.02it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:03<00:05, 2.02it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:04<00:04, 2.02it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:04<00:04, 2.00it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:05<00:04, 1.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:06<00:03, 1.96it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:06<00:03, 1.94it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:07<00:02, 1.93it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:07<00:02, 1.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:08<00:01, 1.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:08<00:01, 1.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:09<00:00, 1.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:09<00:00, 1.95it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:09<00:00, 1.97it/s]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:05, 1.75it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:04, 2.07it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:01<00:03, 2.18it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:01<00:03, 2.24it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:02<00:02, 2.28it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:02<00:02, 2.33it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:03<00:01, 2.34it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:03<00:01, 2.34it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:03<00:00, 2.33it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:04<00:00, 2.33it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.34it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.28it/s]
|
||||
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:53:16 [gpu_model_runner.py:4610] Graph capturing finished in 15 secs, took 0.76 GiB
|
||||
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:53:16 [core.py:259] init engine (profile, create kv cache, warmup model) took 29.33 seconds
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [api_server.py:1099] Supported tasks: ['generate']
|
||||
[0;36m(APIServer pid=69653)[0;0m WARNING 12-19 16:53:17 [model.py:1462] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_responses.py:201] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_chat.py:137] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_completion.py:77] Using default completion sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_chat.py:137] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [api_server.py:1425] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:38] Available routes are:
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /pause, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /resume, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /is_paused, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /metrics, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /health, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /load, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /version, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /ping, Methods: GET
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /ping, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /invocations, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /classify, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /score, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/score, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /rerank, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /pooling, Methods: POST
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: Started server process [69653]
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: Waiting for application startup.
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: Application startup complete.
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:36624 - "GET /v1/models HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:38 [loggers.py:248] Engine 000: Avg prompt throughput: 428.2 tokens/s, Avg generation throughput: 153.1 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:48 [loggers.py:248] Engine 000: Avg prompt throughput: 1264.2 tokens/s, Avg generation throughput: 435.2 tokens/s, Running: 38 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:58 [loggers.py:248] Engine 000: Avg prompt throughput: 925.9 tokens/s, Avg generation throughput: 700.7 tokens/s, Running: 41 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:08 [loggers.py:248] Engine 000: Avg prompt throughput: 643.5 tokens/s, Avg generation throughput: 815.1 tokens/s, Running: 54 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:18 [loggers.py:248] Engine 000: Avg prompt throughput: 662.7 tokens/s, Avg generation throughput: 855.4 tokens/s, Running: 51 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:28 [loggers.py:248] Engine 000: Avg prompt throughput: 978.9 tokens/s, Avg generation throughput: 826.0 tokens/s, Running: 53 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:38 [loggers.py:248] Engine 000: Avg prompt throughput: 936.6 tokens/s, Avg generation throughput: 896.8 tokens/s, Running: 58 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:48 [loggers.py:248] Engine 000: Avg prompt throughput: 480.3 tokens/s, Avg generation throughput: 907.4 tokens/s, Running: 48 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:58 [loggers.py:248] Engine 000: Avg prompt throughput: 919.2 tokens/s, Avg generation throughput: 935.0 tokens/s, Running: 63 reqs, Waiting: 5 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:08 [loggers.py:248] Engine 000: Avg prompt throughput: 627.6 tokens/s, Avg generation throughput: 1008.0 tokens/s, Running: 55 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:18 [loggers.py:248] Engine 000: Avg prompt throughput: 1008.7 tokens/s, Avg generation throughput: 869.9 tokens/s, Running: 50 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 0.8%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:28 [loggers.py:248] Engine 000: Avg prompt throughput: 756.8 tokens/s, Avg generation throughput: 756.5 tokens/s, Running: 45 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 0.8%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:38 [loggers.py:248] Engine 000: Avg prompt throughput: 690.0 tokens/s, Avg generation throughput: 750.2 tokens/s, Running: 44 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.7%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:48 [loggers.py:248] Engine 000: Avg prompt throughput: 757.2 tokens/s, Avg generation throughput: 808.7 tokens/s, Running: 46 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 0.9%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:58 [loggers.py:248] Engine 000: Avg prompt throughput: 840.8 tokens/s, Avg generation throughput: 722.0 tokens/s, Running: 35 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 0.8%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:56:08 [loggers.py:248] Engine 000: Avg prompt throughput: 823.4 tokens/s, Avg generation throughput: 727.0 tokens/s, Running: 45 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 0.8%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:56:18 [loggers.py:248] Engine 000: Avg prompt throughput: 495.1 tokens/s, Avg generation throughput: 806.8 tokens/s, Running: 48 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 0.7%
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60332 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60332 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 498.7602350669986,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 2.004971386433142,
|
||||
"tokens_per_second": 1486.353457790027
|
||||
}
|
||||
+941
@@ -0,0 +1,941 @@
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
WARNING 12-19 17:11:45 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:45 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:45 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'trust_remote_code': True, 'max_model_len': 16384, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 32}
|
||||
[0;36m(APIServer pid=76251)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [model.py:514] Resolved architecture: Qwen3NextForCausalLM
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [model.py:1636] Using max model len 16384
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [config.py:312] Disabling cascade attention since it is not supported for hybrid models.
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [config.py:439] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size.
|
||||
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [config.py:463] Padding mamba page size by 1.49% to ensure that mamba page size and attention page size are exactly equal.
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m INFO 12-19 17:11:53 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m WARNING 12-19 17:11:53 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||
INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl
|
||||
INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl
|
||||
INFO 12-19 17:11:57 [pynccl.py:111] vLLM is using nccl==2.27.3
|
||||
INFO 12-19 17:11:57 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
|
||||
INFO 12-19 17:11:57 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/10 [00:00<?, ?it/s]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 10% Completed | 1/10 [00:03<00:32, 3.65s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 20% Completed | 2/10 [00:07<00:29, 3.64s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 30% Completed | 3/10 [00:11<00:26, 3.79s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 40% Completed | 4/10 [00:15<00:23, 3.97s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 50% Completed | 5/10 [00:18<00:18, 3.76s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 60% Completed | 6/10 [00:22<00:15, 3.85s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 70% Completed | 7/10 [00:26<00:11, 3.83s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 80% Completed | 8/10 [00:30<00:07, 3.84s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 100% Completed | 10/10 [00:34<00:00, 2.97s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Loading safetensors checkpoint shards: 100% Completed | 10/10 [00:34<00:00, 3.46s/it]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:33 [default_loader.py:308] Loading weights took 34.64 seconds
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:34 [gpu_model_runner.py:3659] Model loading took 23.5020 GiB memory and 35.701153 seconds
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:38 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/671837fea6/rank_0_0/backbone for vLLM's torch.compile
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:38 [backends.py:694] Dynamo bytecode transform time: 4.08 s
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:12:40 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:40 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m WARNING 12-19 17:12:41 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m WARNING 12-19 17:12:41 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:43 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 2.59 s
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:43 [monitor.py:34] torch.compile takes 6.67 s in total
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m WARNING 12-19 17:12:43 [decorators.py:528] Cannot save aot compilation to path /home/kyuz0/.cache/vllm/torch_aot_compile/8cfc5b4a85b0195581f74785744c13daeda3dd4bc8885cc84b5001d538a63819/rank_1_0/model, error:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m WARNING 12-19 17:12:43 [decorators.py:528] Cannot save aot compilation to path /home/kyuz0/.cache/vllm/torch_aot_compile/8cfc5b4a85b0195581f74785744c13daeda3dd4bc8885cc84b5001d538a63819/rank_0_0/model, error:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:44 [gpu_worker.py:375] Available KV cache memory: 7.09 GiB
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m INFO 12-19 17:12:45 [kv_cache_utils.py:1291] GPU KV cache size: 154,496 tokens
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m INFO 12-19 17:12:45 [kv_cache_utils.py:1296] Maximum concurrency for 16,384 tokens per request: 33.47x
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/11 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 9%|▉ | 1/11 [00:00<00:05, 1.95it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 18%|█▊ | 2/11 [00:01<00:04, 1.83it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 3/11 [00:01<00:04, 1.91it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▋ | 4/11 [00:02<00:03, 1.93it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 5/11 [00:02<00:03, 1.95it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 6/11 [00:03<00:02, 1.95it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 64%|██████▎ | 7/11 [00:03<00:02, 1.97it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 73%|███████▎ | 8/11 [00:04<00:01, 1.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 9/11 [00:04<00:01, 1.95it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 91%|█████████ | 10/11 [00:05<00:00, 1.92it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:05<00:00, 1.98it/s]
|
||||
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:05<00:00, 1.95it/s]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] EngineCore failed to start.
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] Traceback (most recent call last):
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] super().__init__(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] vllm_config
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] self.collective_rpc("compile_or_warm_up_model")
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] return aggregate(get_response())
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] raise RuntimeError(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ...<2 lines>...
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] RuntimeError: Worker failed with error 'at 106:17:
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] if USE_ALIBI_SLOPES:
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] alibi_slope = tl.load(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866]
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866]
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] # iterate through tiles
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] for j in range(0, num_blocks):
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866]
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] arange's range must be a power of 2', please check the stack trace above for the root cause
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m Process EngineCore_DP0:
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m Traceback (most recent call last):
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m self.run()
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m self._target(*self._args, **self._kwargs)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m raise e
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m engine_core = EngineCoreProc(*args, **kwargs)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m super().__init__(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m vllm_config, executor_class, log_stats, executor_fail_callback
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m vllm_config
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m self.model_executor.initialize_from_config(kv_cache_configs)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m self.collective_rpc("compile_or_warm_up_model")
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m return aggregate(get_response())
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~^^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m raise RuntimeError(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ...<2 lines>...
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m RuntimeError: Worker failed with error 'at 106:17:
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m if USE_ALIBI_SLOPES:
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m alibi_slope = tl.load(
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m # iterate through tiles
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m for j in range(0, num_blocks):
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m offs_n = tl.arange(0, BLOCK_SIZE)
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^
|
||||
[0;36m(EngineCore_DP0 pid=76413)[0;0m arange's range must be a power of 2', please check the stack trace above for the root cause
|
||||
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||
[0;36m(APIServer pid=76251)[0;0m Traceback (most recent call last):
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/bin/vllm", line 7, in <module>
|
||||
[0;36m(APIServer pid=76251)[0;0m sys.exit(main())
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
|
||||
[0;36m(APIServer pid=76251)[0;0m args.dispatch_function(args)
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
|
||||
[0;36m(APIServer pid=76251)[0;0m uvloop.run(run_server(args))
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run
|
||||
[0;36m(APIServer pid=76251)[0;0m return __asyncio.run(
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=76251)[0;0m wrapper(),
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m ...<2 lines>...
|
||||
[0;36m(APIServer pid=76251)[0;0m **run_kwargs
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m )
|
||||
[0;36m(APIServer pid=76251)[0;0m ^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run
|
||||
[0;36m(APIServer pid=76251)[0;0m return runner.run(main)
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run
|
||||
[0;36m(APIServer pid=76251)[0;0m return self._loop.run_until_complete(task)
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper
|
||||
[0;36m(APIServer pid=76251)[0;0m return await main
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server
|
||||
[0;36m(APIServer pid=76251)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker
|
||||
[0;36m(APIServer pid=76251)[0;0m async with build_async_engine_client(
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=76251)[0;0m args,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m client_config=client_config,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m ) as engine_client:
|
||||
[0;36m(APIServer pid=76251)[0;0m ^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||
[0;36m(APIServer pid=76251)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client
|
||||
[0;36m(APIServer pid=76251)[0;0m async with build_async_engine_client_from_engine_args(
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=76251)[0;0m engine_args,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m ...<2 lines>...
|
||||
[0;36m(APIServer pid=76251)[0;0m client_config=client_config,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m ) as engine:
|
||||
[0;36m(APIServer pid=76251)[0;0m ^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||
[0;36m(APIServer pid=76251)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args
|
||||
[0;36m(APIServer pid=76251)[0;0m async_llm = AsyncLLM.from_vllm_config(
|
||||
[0;36m(APIServer pid=76251)[0;0m vllm_config=vllm_config,
|
||||
[0;36m(APIServer pid=76251)[0;0m ...<6 lines>...
|
||||
[0;36m(APIServer pid=76251)[0;0m client_index=client_index,
|
||||
[0;36m(APIServer pid=76251)[0;0m )
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config
|
||||
[0;36m(APIServer pid=76251)[0;0m return cls(
|
||||
[0;36m(APIServer pid=76251)[0;0m vllm_config=vllm_config,
|
||||
[0;36m(APIServer pid=76251)[0;0m ...<9 lines>...
|
||||
[0;36m(APIServer pid=76251)[0;0m client_index=client_index,
|
||||
[0;36m(APIServer pid=76251)[0;0m )
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
|
||||
[0;36m(APIServer pid=76251)[0;0m self.engine_core = EngineCoreClient.make_async_mp_client(
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=76251)[0;0m vllm_config=vllm_config,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m ...<4 lines>...
|
||||
[0;36m(APIServer pid=76251)[0;0m client_index=client_index,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=76251)[0;0m )
|
||||
[0;36m(APIServer pid=76251)[0;0m ^
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client
|
||||
[0;36m(APIServer pid=76251)[0;0m return AsyncMPClient(*client_args)
|
||||
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__
|
||||
[0;36m(APIServer pid=76251)[0;0m super().__init__(
|
||||
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~^
|
||||
[0;36m(APIServer pid=76251)[0;0m asyncio_mode=True,
|
||||
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 14:23:22 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7c39b23740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af26c427-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38794 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.66 \nPeak output token throughput (tok/s): 398.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 392.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.71 \nMedian TTFT (ms): 61.84 \nP99 TTFT (ms): 180.02 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.85 \nMedian TPOT (ms): 30.79 \nP99 TPOT (ms): 33.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.86 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 55.30 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 14:26:47 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ef7ca647740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-65595a03-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.73 \nTotal input tokens: 145810 \nTotal generated tokens: 152105 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 742.97 \nPeak output token throughput (tok/s): 1252.00 \nPeak concurrent requests: 55.00 \nTotal token throughput (tok/s): 1455.19 \n---------------Time to First Token----------------\nMean TTFT (ms): 81.58 \nMedian TTFT (ms): 70.11 \nP99 TTFT (ms): 184.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.28 \nMedian TPOT (ms): 36.83 \nP99 TPOT (ms): 50.83 \n---------------Inter-token Latency----------------\nMean ITL (ms): 36.98 \nMedian ITL (ms): 34.37 \nP99 ITL (ms): 119.18 \n==================================================\n"
|
||||
}
|
||||
+1027
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 389.5727865589979,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 736330,
|
||||
"requests_per_second": 2.5669144111239337,
|
||||
"tokens_per_second": 1890.096088342886
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:41:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f794120f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-819f7ea7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 189.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.95 \nOutput token throughput (tok/s): 204.80 \nPeak output token throughput (tok/s): 415.00 \nPeak concurrent requests: 10.00 \nTotal token throughput (tok/s): 404.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.83 \nMedian TTFT (ms): 50.95 \nP99 TTFT (ms): 232.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 19.52 \nMedian TPOT (ms): 19.18 \nP99 TPOT (ms): 24.48 \n---------------Inter-token Latency----------------\nMean ITL (ms): 19.42 \nMedian ITL (ms): 18.84 \nP99 ITL (ms): 34.36 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 15:44:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f84e0897740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-9c7b74a1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.04 \nTotal input tokens: 145810 \nTotal generated tokens: 152194 \nRequest throughput (req/s): 3.67 \nOutput token throughput (tok/s): 776.36 \nPeak output token throughput (tok/s): 1303.00 \nPeak concurrent requests: 46.00 \nTotal token throughput (tok/s): 1520.15 \n---------------Time to First Token----------------\nMean TTFT (ms): 86.17 \nMedian TTFT (ms): 57.78 \nP99 TTFT (ms): 291.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.21 \nMedian TPOT (ms): 27.26 \nP99 TPOT (ms): 51.74 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.56 \nMedian ITL (ms): 23.38 \nP99 ITL (ms): 140.32 \n==================================================\n"
|
||||
}
|
||||
+1034
File diff suppressed because it is too large
Load Diff
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 14:42:05 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f57a3c9f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3e31d4e4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 215.02 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.28 \nPeak output token throughput (tok/s): 324.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 362.52 \n---------------Time to First Token----------------\nMean TTFT (ms): 118.15 \nMedian TTFT (ms): 106.93 \nP99 TTFT (ms): 227.27 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.56 \nMedian TPOT (ms): 55.08 \nP99 TPOT (ms): 69.46 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.82 \nMedian ITL (ms): 54.23 \nP99 ITL (ms): 120.82 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "WARNING 12-19 14:45:51 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7eff5fdd7740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84255adf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 238.83 \nTotal input tokens: 145540 \nTotal generated tokens: 151955 \nRequest throughput (req/s): 3.01 \nOutput token throughput (tok/s): 636.24 \nPeak output token throughput (tok/s): 960.00 \nPeak concurrent requests: 94.00 \nTotal token throughput (tok/s): 1245.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 1170.97 \nMedian TTFT (ms): 182.43 \nP99 TTFT (ms): 5964.07 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.13 \nMedian TPOT (ms): 74.02 \nP99 TPOT (ms): 87.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 72.97 \nMedian ITL (ms): 68.87 \nP99 ITL (ms): 158.23 \n==================================================\n"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 604.6067389169984,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 738792,
|
||||
"requests_per_second": 1.6539676712688476,
|
||||
"tokens_per_second": 1221.9380837920544
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f9e1c29aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e4e803cf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.63 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.85 \nPeak output token throughput (tok/s): 350.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 392.04 \n---------------Time to First Token----------------\nMean TTFT (ms): 100.59 \nMedian TTFT (ms): 84.87 \nP99 TTFT (ms): 217.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.46 \nMedian TPOT (ms): 41.30 \nP99 TPOT (ms): 45.56 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.46 \nMedian ITL (ms): 40.41 \nP99 ITL (ms): 81.46 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5f5b1aeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2d3278b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 211.85 \nTotal input tokens: 146694 \nTotal generated tokens: 155558 \nRequest throughput (req/s): 3.40 \nOutput token throughput (tok/s): 734.30 \nPeak output token throughput (tok/s): 1241.00 \nPeak concurrent requests: 68.00 \nTotal Token throughput (tok/s): 1426.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 111.24 \nMedian TTFT (ms): 89.87 \nP99 TTFT (ms): 307.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 52.80 \nMedian TPOT (ms): 52.89 \nP99 TPOT (ms): 76.85 \n---------------Inter-token Latency----------------\nMean ITL (ms): 52.36 \nMedian ITL (ms): 48.02 \nP99 ITL (ms): 172.90 \n==================================================\n"
|
||||
}
|
||||
+1042
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 499.5056346299998,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 2.00197941859201,
|
||||
"tokens_per_second": 1484.1354103024892
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 680.202111824,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 1.4701512721247574,
|
||||
"tokens_per_second": 1089.8731231693348
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fd4aef3aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fce2ee75-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.29 \nTotal input tokens: 37841 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.79 \nPeak output token throughput (tok/s): 406.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 392.56 \n---------------Time to First Token----------------\nMean TTFT (ms): 73.39 \nMedian TTFT (ms): 61.33 \nP99 TTFT (ms): 185.43 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.73 \nMedian TPOT (ms): 30.60 \nP99 TPOT (ms): 33.54 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.74 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 50.56 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f082b14aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af5860ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.38 \nTotal input tokens: 145810 \nTotal generated tokens: 152213 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 744.74 \nPeak output token throughput (tok/s): 1238.00 \nPeak concurrent requests: 53.00 \nTotal Token throughput (tok/s): 1458.16 \n---------------Time to First Token----------------\nMean TTFT (ms): 75.15 \nMedian TTFT (ms): 63.63 \nP99 TTFT (ms): 179.56 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 35.83 \nMedian TPOT (ms): 35.55 \nP99 TPOT (ms): 48.38 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.60 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 112.07 \n==================================================\n"
|
||||
}
|
||||
+1040
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 354.4166975950002,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 736330,
|
||||
"requests_per_second": 2.821537491844479,
|
||||
"tokens_per_second": 2077.5827013698454
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc4f7b12ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1dd28fd7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 214.85 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.43 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 19.00 \nTotal Token throughput (tok/s): 362.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 115.51 \nMedian TTFT (ms): 108.55 \nP99 TTFT (ms): 235.09 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 53.14 \nMedian TPOT (ms): 53.86 \nP99 TPOT (ms): 69.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 53.53 \nMedian ITL (ms): 52.81 \nP99 ITL (ms): 120.56 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f58e6eeeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f4f4694-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 236.39 \nTotal input tokens: 145540 \nTotal generated tokens: 152178 \nRequest throughput (req/s): 3.05 \nOutput token throughput (tok/s): 643.77 \nPeak output token throughput (tok/s): 1024.00 \nPeak concurrent requests: 92.00 \nTotal Token throughput (tok/s): 1259.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 850.20 \nMedian TTFT (ms): 141.29 \nP99 TTFT (ms): 5024.92 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.13 \nMedian TPOT (ms): 71.70 \nP99 TPOT (ms): 89.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.90 \nMedian ITL (ms): 66.80 \nP99 ITL (ms): 157.64 \n==================================================\n"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 602.7616506659997,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 738792,
|
||||
"requests_per_second": 1.6590305619063292,
|
||||
"tokens_per_second": 1225.6785068919007
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f84fa08eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84569f25-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 203.97 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.88 \nOutput token throughput (tok/s): 197.56 \nPeak output token throughput (tok/s): 330.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 385.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 108.91 \nMedian TTFT (ms): 95.15 \nP99 TTFT (ms): 230.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.90 \nMedian TPOT (ms): 47.66 \nP99 TPOT (ms): 53.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.94 \nMedian ITL (ms): 46.64 \nP99 ITL (ms): 85.76 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5366d42ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a714d425-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 217.26 \nTotal input tokens: 146694 \nTotal generated tokens: 155585 \nRequest throughput (req/s): 3.31 \nOutput token throughput (tok/s): 716.13 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 77.00 \nTotal Token throughput (tok/s): 1391.33 \n---------------Time to First Token----------------\nMean TTFT (ms): 186.47 \nMedian TTFT (ms): 105.59 \nP99 TTFT (ms): 1603.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 59.68 \nMedian TPOT (ms): 59.91 \nP99 TPOT (ms): 84.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 59.35 \nMedian ITL (ms): 54.32 \nP99 ITL (ms): 190.63 \n==================================================\n"
|
||||
}
|
||||
+1047
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 544.9470995769998,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 1.8350405035208417,
|
||||
"tokens_per_second": 1360.3779166371198
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fde305aeca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a3fea59e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.99 \nTotal input tokens: 40429 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 197.07 \nPeak output token throughput (tok/s): 340.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 402.31 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.59 \nMedian TTFT (ms): 82.18 \nP99 TTFT (ms): 271.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 38.86 \nMedian TPOT (ms): 38.92 \nP99 TPOT (ms): 50.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.20 \nMedian ITL (ms): 32.74 \nP99 ITL (ms): 86.49 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f83b1b0aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f36abb16-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 241.29 \nTotal input tokens: 158086 \nTotal generated tokens: 152183 \nRequest throughput (req/s): 2.98 \nOutput token throughput (tok/s): 630.70 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 152.00 \nTotal Token throughput (tok/s): 1285.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 12436.56 \nMedian TTFT (ms): 17115.90 \nP99 TTFT (ms): 24750.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 84.04 \nMedian TPOT (ms): 85.32 \nP99 TPOT (ms): 115.82 \n---------------Inter-token Latency----------------\nMean ITL (ms): 83.72 \nMedian ITL (ms): 75.54 \nP99 ITL (ms): 261.14 \n==================================================\n"
|
||||
}
|
||||
+1048
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 878.6614384089999,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 755432,
|
||||
"requests_per_second": 1.1380947840509639,
|
||||
"tokens_per_second": 859.7532189051877
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f87f9f3eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a1348ed0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.66 \nTotal input tokens: 40429 \nTotal generated tokens: 39127 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 205.22 \nPeak output token throughput (tok/s): 422.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 417.26 \n---------------Time to First Token----------------\nMean TTFT (ms): 89.11 \nMedian TTFT (ms): 64.29 \nP99 TTFT (ms): 255.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.00 \nMedian TPOT (ms): 21.60 \nP99 TPOT (ms): 28.81 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.91 \nMedian ITL (ms): 20.88 \nP99 ITL (ms): 45.88 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8e7772aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-370f24dd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.05 \nTotal input tokens: 158086 \nTotal generated tokens: 153200 \nRequest throughput (req/s): 3.55 \nOutput token throughput (tok/s): 754.51 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 78.00 \nTotal Token throughput (tok/s): 1533.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 222.29 \nMedian TTFT (ms): 103.65 \nP99 TTFT (ms): 2503.58 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.74 \nMedian TPOT (ms): 55.60 \nP99 TPOT (ms): 83.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.25 \nMedian ITL (ms): 49.31 \nP99 ITL (ms): 204.91 \n==================================================\n"
|
||||
}
|
||||
+1074
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 569.8341109329999,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 755432,
|
||||
"requests_per_second": 1.7548966985543941,
|
||||
"tokens_per_second": 1325.705122782343
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fab82852ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb2c00d0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.25 \nTotal input tokens: 40429 \nTotal generated tokens: 39398 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 194.80 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 17.00 \nTotal Token throughput (tok/s): 394.70 \n---------------Time to First Token----------------\nMean TTFT (ms): 135.44 \nMedian TTFT (ms): 102.54 \nP99 TTFT (ms): 390.74 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.24 \nMedian TPOT (ms): 46.60 \nP99 TPOT (ms): 56.08 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.00 \nMedian ITL (ms): 45.58 \nP99 ITL (ms): 140.61 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f310eb62ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb0dc17a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 320.24 \nTotal input tokens: 158086 \nTotal generated tokens: 152563 \nRequest throughput (req/s): 2.25 \nOutput token throughput (tok/s): 476.40 \nPeak output token throughput (tok/s): 608.00 \nPeak concurrent requests: 304.00 \nTotal Token throughput (tok/s): 970.05 \n---------------Time to First Token----------------\nMean TTFT (ms): 52928.91 \nMedian TTFT (ms): 59992.28 \nP99 TTFT (ms): 104765.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 61.40 \nMedian TPOT (ms): 60.27 \nP99 TPOT (ms): 90.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 60.58 \nMedian ITL (ms): 53.74 \nP99 ITL (ms): 244.74 \n==================================================\n"
|
||||
}
|
||||
+1092
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 767.3010607799997,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 755432,
|
||||
"requests_per_second": 1.303269408989804,
|
||||
"tokens_per_second": 984.5314161719857
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f575fb42ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1e0af39c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40157 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 202.57 \nPeak output token throughput (tok/s): 311.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 396.07 \n---------------Time to First Token----------------\nMean TTFT (ms): 120.83 \nMedian TTFT (ms): 107.92 \nP99 TTFT (ms): 215.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.17 \nMedian TPOT (ms): 44.69 \nP99 TPOT (ms): 56.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.94 \nMedian ITL (ms): 42.97 \nP99 ITL (ms): 110.51 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f193f9faca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5a7df568-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.37 \nTotal input tokens: 146694 \nTotal generated tokens: 153158 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 614.17 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 168.00 \nTotal Token throughput (tok/s): 1202.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 14118.91 \nMedian TTFT (ms): 18607.33 \nP99 TTFT (ms): 31435.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.10 \nMedian TPOT (ms): 90.68 \nP99 TPOT (ms): 104.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.93 \nMedian ITL (ms): 84.69 \nP99 ITL (ms): 190.09 \n==================================================\n"
|
||||
}
|
||||
+1054
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 726.8079903329999,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 1.3758792051004176,
|
||||
"tokens_per_second": 1019.986034633913
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6885e56ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df1894e5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.60 \nTotal input tokens: 38358 \nTotal generated tokens: 39514 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 206.23 \nPeak output token throughput (tok/s): 361.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 406.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.14 \nMedian TTFT (ms): 66.00 \nP99 TTFT (ms): 145.78 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.00 \nMedian TPOT (ms): 28.14 \nP99 TPOT (ms): 35.12 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.59 \nMedian ITL (ms): 27.66 \nP99 ITL (ms): 54.69 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fb8a2696ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e434292b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.64 \nTotal input tokens: 146694 \nTotal generated tokens: 153347 \nRequest throughput (req/s): 3.54 \nOutput token throughput (tok/s): 753.01 \nPeak output token throughput (tok/s): 1160.00 \nPeak concurrent requests: 65.00 \nTotal Token throughput (tok/s): 1473.36 \n---------------Time to First Token----------------\nMean TTFT (ms): 93.13 \nMedian TTFT (ms): 87.23 \nP99 TTFT (ms): 181.31 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.81 \nMedian TPOT (ms): 47.62 \nP99 TPOT (ms): 61.13 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.43 \nMedian ITL (ms): 44.70 \nP99 ITL (ms): 109.90 \n==================================================\n"
|
||||
}
|
||||
+1076
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 449.04138348200104,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 2.2269662369327774,
|
||||
"tokens_per_second": 1650.9257882903235
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f98e81ceca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b53e2ad8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.26 \nTotal input tokens: 38358 \nTotal generated tokens: 38404 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 195.68 \nPeak output token throughput (tok/s): 304.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 391.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 519.96 \nMedian TTFT (ms): 124.31 \nP99 TTFT (ms): 10120.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 44.04 \nMedian TPOT (ms): 42.30 \nP99 TPOT (ms): 57.88 \n---------------Inter-token Latency----------------\nMean ITL (ms): 43.67 \nMedian ITL (ms): 40.54 \nP99 ITL (ms): 125.33 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4ab18f6ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-72f2dd3c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 351.46 \nTotal input tokens: 146694 \nTotal generated tokens: 148389 \nRequest throughput (req/s): 2.05 \nOutput token throughput (tok/s): 422.20 \nPeak output token throughput (tok/s): 544.00 \nPeak concurrent requests: 369.00 \nTotal Token throughput (tok/s): 839.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 72209.01 \nMedian TTFT (ms): 79268.59 \nP99 TTFT (ms): 146128.91 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.26 \nMedian TPOT (ms): 71.50 \nP99 TPOT (ms): 93.20 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.85 \nMedian ITL (ms): 62.03 \nP99 ITL (ms): 206.26 \n==================================================\n"
|
||||
}
|
||||
+1128
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 1089.8343978980001,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 0.9175705978162676,
|
||||
"tokens_per_second": 680.226281561525
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f532a48eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df39931f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.49 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 195.98 \nPeak output token throughput (tok/s): 365.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 386.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.34 \nMedian TTFT (ms): 69.07 \nP99 TTFT (ms): 180.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.10 \nMedian TPOT (ms): 37.08 \nP99 TPOT (ms): 40.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 37.10 \nMedian ITL (ms): 36.27 \nP99 ITL (ms): 58.47 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f081ef62ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ba51e591-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 719 \nFailed requests: 1 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 209.75 \nTotal input tokens: 145579 \nTotal generated tokens: 150640 \nRequest throughput (req/s): 3.43 \nOutput token throughput (tok/s): 718.18 \nPeak output token throughput (tok/s): 1259.00 \nPeak concurrent requests: 58.00 \nTotal Token throughput (tok/s): 1412.23 \n---------------Time to First Token----------------\nMean TTFT (ms): 85.92 \nMedian TTFT (ms): 75.54 \nP99 TTFT (ms): 188.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 42.81 \nMedian TPOT (ms): 42.63 \nP99 TPOT (ms): 54.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 42.63 \nMedian ITL (ms): 40.26 \nP99 ITL (ms): 125.13 \n==================================================\n"
|
||||
}
|
||||
+1043
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 410.0767225319996,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 736330,
|
||||
"requests_per_second": 2.438568065569649,
|
||||
"tokens_per_second": 1795.5908237208996
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4b1de6aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-073f047f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.90 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 203.07 \nPeak output token throughput (tok/s): 413.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 401.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.57 \nMedian TTFT (ms): 47.42 \nP99 TTFT (ms): 140.81 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 21.86 \nMedian TPOT (ms): 21.74 \nP99 TPOT (ms): 23.96 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.82 \nMedian ITL (ms): 21.46 \nP99 ITL (ms): 32.13 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f791efdeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2ef19409-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.59 \nTotal input tokens: 145810 \nTotal generated tokens: 151842 \nRequest throughput (req/s): 3.66 \nOutput token throughput (tok/s): 772.38 \nPeak output token throughput (tok/s): 1274.00 \nPeak concurrent requests: 43.00 \nTotal Token throughput (tok/s): 1514.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 57.73 \nMedian TTFT (ms): 49.35 \nP99 TTFT (ms): 135.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.29 \nMedian TPOT (ms): 24.90 \nP99 TPOT (ms): 34.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.09 \nMedian ITL (ms): 23.53 \nP99 ITL (ms): 78.49 \n==================================================\n"
|
||||
}
|
||||
+1065
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 269.04666597899995,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 736330,
|
||||
"requests_per_second": 3.7168273257028708,
|
||||
"tokens_per_second": 2736.811464734795
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fa6fac76ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84df0cbe-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 217.42 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.83 \nOutput token throughput (tok/s): 180.27 \nPeak output token throughput (tok/s): 290.00 \nPeak concurrent requests: 20.00 \nTotal Token throughput (tok/s): 358.52 \n---------------Time to First Token----------------\nMean TTFT (ms): 129.43 \nMedian TTFT (ms): 121.63 \nP99 TTFT (ms): 431.05 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 61.28 \nMedian TPOT (ms): 61.83 \nP99 TPOT (ms): 74.53 \n---------------Inter-token Latency----------------\nMean ITL (ms): 61.80 \nMedian ITL (ms): 59.99 \nP99 ITL (ms): 129.94 \n==================================================\n"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fdfa3a9eca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ea75820-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 246.91 \nTotal input tokens: 145540 \nTotal generated tokens: 152007 \nRequest throughput (req/s): 2.92 \nOutput token throughput (tok/s): 615.63 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 109.00 \nTotal Token throughput (tok/s): 1205.06 \n---------------Time to First Token----------------\nMean TTFT (ms): 4078.97 \nMedian TTFT (ms): 3857.18 \nP99 TTFT (ms): 10112.65 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 78.49 \nMedian TPOT (ms): 79.02 \nP99 TPOT (ms): 93.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 78.52 \nMedian ITL (ms): 74.18 \nP99 ITL (ms): 163.52 \n==================================================\n"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 674.5005297629996,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 738792,
|
||||
"requests_per_second": 1.4825785242175744,
|
||||
"tokens_per_second": 1095.31715306375
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f16adb6eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7a645b9b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 199.44 \nTotal input tokens: 38756 \nTotal generated tokens: 38779 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 194.44 \nPeak output token throughput (tok/s): 395.00 \nPeak concurrent requests: 14.00 \nTotal Token throughput (tok/s): 388.76 \n---------------Time to First Token----------------\nMean TTFT (ms): 75.40 \nMedian TTFT (ms): 67.78 \nP99 TTFT (ms): 192.76 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.60 \nMedian TPOT (ms): 28.16 \nP99 TPOT (ms): 41.84 \n---------------Inter-token Latency----------------\nMean ITL (ms): 29.20 \nMedian ITL (ms): 25.34 \nP99 ITL (ms): 71.24 \n==================================================\n"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": false,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f64d27beca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c0304eb1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nERROR 12-09 20:35:02 [repo_utils.py:65] Error retrieving file list: 502 Server Error: Bad Gateway for url: https://huggingface.co/api/models/openai/gpt-oss-20b/tree/main?recursive=True&expand=False, retrying 1 of 2\nERROR 12-09 20:35:04 [repo_utils.py:63] Error retrieving file list: 502 Server Error: Bad Gateway for url: https://huggingface.co/api/models/openai/gpt-oss-20b/tree/main?recursive=True&expand=False\nERROR 12-09 20:35:04 [repo_utils.py:128] Error retrieving file list. Please ensure your `model_name_or_path``repo_type`, `token` and `revision` arguments are correctly set. Returning an empty list.\n"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 393.24469268199937,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 738792,
|
||||
"requests_per_second": 2.542945953522781,
|
||||
"tokens_per_second": 1878.7081268950026
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
WARNING 12-10 09:53:36 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:36 [api_server.py:1772] vLLM API server version 0.12.0
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:36 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'host': '127.0.0.1', 'model': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'trust_remote_code': True, 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64}
|
||||
[0;36m(APIServer pid=4288)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:47 [model.py:637] Resolved architecture: LlamaForCausalLM
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:47 [model.py:1750] Using max model len 65536
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:47 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
[0;36m(APIServer pid=4288)[0;0m Traceback (most recent call last):
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 409, in hf_raise_for_status
|
||||
[0;36m(APIServer pid=4288)[0;0m response.raise_for_status()
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/requests/models.py", line 1026, in raise_for_status
|
||||
[0;36m(APIServer pid=4288)[0;0m raise HTTPError(http_error_msg, response=self)
|
||||
[0;36m(APIServer pid=4288)[0;0m requests.exceptions.HTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block
|
||||
[0;36m(APIServer pid=4288)[0;0m
|
||||
[0;36m(APIServer pid=4288)[0;0m The above exception was the direct cause of the following exception:
|
||||
[0;36m(APIServer pid=4288)[0;0m
|
||||
[0;36m(APIServer pid=4288)[0;0m Traceback (most recent call last):
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/bin/vllm", line 7, in <module>
|
||||
[0;36m(APIServer pid=4288)[0;0m sys.exit(main())
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
|
||||
[0;36m(APIServer pid=4288)[0;0m args.dispatch_function(args)
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
|
||||
[0;36m(APIServer pid=4288)[0;0m uvloop.run(run_server(args))
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run
|
||||
[0;36m(APIServer pid=4288)[0;0m return __asyncio.run(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/asyncio/runners.py", line 195, in run
|
||||
[0;36m(APIServer pid=4288)[0;0m return runner.run(main)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/asyncio/runners.py", line 118, in run
|
||||
[0;36m(APIServer pid=4288)[0;0m return self._loop.run_until_complete(task)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper
|
||||
[0;36m(APIServer pid=4288)[0;0m return await main
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1819, in run_server
|
||||
[0;36m(APIServer pid=4288)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1838, in run_server_worker
|
||||
[0;36m(APIServer pid=4288)[0;0m async with build_async_engine_client(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
|
||||
[0;36m(APIServer pid=4288)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 183, in build_async_engine_client
|
||||
[0;36m(APIServer pid=4288)[0;0m async with build_async_engine_client_from_engine_args(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
|
||||
[0;36m(APIServer pid=4288)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 224, in build_async_engine_client_from_engine_args
|
||||
[0;36m(APIServer pid=4288)[0;0m async_llm = AsyncLLM.from_vllm_config(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 223, in from_vllm_config
|
||||
[0;36m(APIServer pid=4288)[0;0m return cls(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 114, in __init__
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = init_tokenizer_from_config(self.model_config)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 227, in init_tokenizer_from_config
|
||||
[0;36m(APIServer pid=4288)[0;0m return get_tokenizer(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 191, in get_tokenizer
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = TokenizerRegistry.get_tokenizer(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 86, in get_tokenizer
|
||||
[0;36m(APIServer pid=4288)[0;0m return item.from_pretrained(*args, **kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/hf.py", line 84, in from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = AutoTokenizer.from_pretrained(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 1156, in from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2113, in from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m return cls._from_pretrained(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2395, in _from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = cls._patch_mistral_regex(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2438, in _patch_mistral_regex
|
||||
[0;36m(APIServer pid=4288)[0;0m if _is_local or is_base_mistral(pretrained_model_name_or_path):
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2432, in is_base_mistral
|
||||
[0;36m(APIServer pid=4288)[0;0m model = model_info(model_id)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
|
||||
[0;36m(APIServer pid=4288)[0;0m return fn(*args, **kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/hf_api.py", line 2638, in model_info
|
||||
[0;36m(APIServer pid=4288)[0;0m hf_raise_for_status(r)
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 482, in hf_raise_for_status
|
||||
[0;36m(APIServer pid=4288)[0;0m raise _format(HfHubHTTPError, str(e), response) from e
|
||||
[0;36m(APIServer pid=4288)[0;0m huggingface_hub.errors.HfHubHTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73dcacabf4c0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f177f1fc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 210.71 \nPeak output token throughput (tok/s): 430.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 411.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 145.79 \nMedian TTFT (ms): 110.27 \nP99 TTFT (ms): 399.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.46 \nMedian TPOT (ms): 22.67 \nP99 TPOT (ms): 36.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.37 \nMedian ITL (ms): 21.34 \nP99 ITL (ms): 84.18 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cabb891f240>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31b9a516-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 215.08 \nTotal input tokens: 146694 \nTotal generated tokens: 155647 \nRequest throughput (req/s): 3.35 \nOutput token throughput (tok/s): 723.67 \nPeak output token throughput (tok/s): 1208.00 \nPeak concurrent requests: 113.00 \nTotal Token throughput (tok/s): 1405.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 11329.86 \nMedian TTFT (ms): 14049.92 \nP99 TTFT (ms): 20450.77 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 40.85 \nMedian TPOT (ms): 39.82 \nP99 TPOT (ms): 83.14 \n---------------Inter-token Latency----------------\nMean ITL (ms): 40.32 \nMedian ITL (ms): 26.70 \nP99 ITL (ms): 338.67 \n==================================================\n"
|
||||
}
|
||||
+1027
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user