diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Llama-3.1-8B-Instruct-FP8-block_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Llama-3.1-8B-Instruct-FP8-block_tp1_server.log deleted file mode 100644 index fe392f1..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Llama-3.1-8B-Instruct-FP8-block_tp1_server.log +++ /dev/null @@ -1,95 +0,0 @@ -WARNING 12-10 09:53:36 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=4288) INFO 12-10 09:53:36 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=4288) INFO 12-10 09:53:36 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'host': '127.0.0.1', 'model': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'trust_remote_code': True, 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=4288) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=4288) INFO 12-10 09:53:47 [model.py:637] Resolved architecture: LlamaForCausalLM -(APIServer pid=4288) INFO 12-10 09:53:47 [model.py:1750] Using max model len 65536 -(APIServer pid=4288) INFO 12-10 09:53:47 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(APIServer pid=4288) Traceback (most recent call last): -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 409, in hf_raise_for_status -(APIServer pid=4288) response.raise_for_status() -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/requests/models.py", line 1026, in raise_for_status -(APIServer pid=4288) raise HTTPError(http_error_msg, response=self) -(APIServer pid=4288) requests.exceptions.HTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block -(APIServer pid=4288) -(APIServer pid=4288) The above exception was the direct cause of the following exception: -(APIServer pid=4288) -(APIServer pid=4288) Traceback (most recent call last): -(APIServer pid=4288) File "/venv/main/bin/vllm", line 7, in -(APIServer pid=4288) sys.exit(main()) -(APIServer pid=4288) ^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main -(APIServer pid=4288) args.dispatch_function(args) -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd -(APIServer pid=4288) uvloop.run(run_server(args)) -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run -(APIServer pid=4288) return __asyncio.run( -(APIServer pid=4288) ^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/asyncio/runners.py", line 195, in run -(APIServer pid=4288) return runner.run(main) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/asyncio/runners.py", line 118, in run -(APIServer pid=4288) return self._loop.run_until_complete(task) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper -(APIServer pid=4288) return await main -(APIServer pid=4288) ^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1819, in run_server -(APIServer pid=4288) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1838, in run_server_worker -(APIServer pid=4288) async with build_async_engine_client( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__ -(APIServer pid=4288) return await anext(self.gen) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 183, in build_async_engine_client -(APIServer pid=4288) async with build_async_engine_client_from_engine_args( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__ -(APIServer pid=4288) return await anext(self.gen) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 224, in build_async_engine_client_from_engine_args -(APIServer pid=4288) async_llm = AsyncLLM.from_vllm_config( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 223, in from_vllm_config -(APIServer pid=4288) return cls( -(APIServer pid=4288) ^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 114, in __init__ -(APIServer pid=4288) tokenizer = init_tokenizer_from_config(self.model_config) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 227, in init_tokenizer_from_config -(APIServer pid=4288) return get_tokenizer( -(APIServer pid=4288) ^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 191, in get_tokenizer -(APIServer pid=4288) tokenizer = TokenizerRegistry.get_tokenizer( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 86, in get_tokenizer -(APIServer pid=4288) return item.from_pretrained(*args, **kwargs) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/hf.py", line 84, in from_pretrained -(APIServer pid=4288) tokenizer = AutoTokenizer.from_pretrained( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 1156, in from_pretrained -(APIServer pid=4288) return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2113, in from_pretrained -(APIServer pid=4288) return cls._from_pretrained( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2395, in _from_pretrained -(APIServer pid=4288) tokenizer = cls._patch_mistral_regex( -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2438, in _patch_mistral_regex -(APIServer pid=4288) if _is_local or is_base_mistral(pretrained_model_name_or_path): -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2432, in is_base_mistral -(APIServer pid=4288) model = model_info(model_id) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn -(APIServer pid=4288) return fn(*args, **kwargs) -(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^ -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/hf_api.py", line 2638, in model_info -(APIServer pid=4288) hf_raise_for_status(r) -(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 482, in hf_raise_for_status -(APIServer pid=4288) raise _format(HfHubHTTPError, str(e), response) from e -(APIServer pid=4288) huggingface_hub.errors.HfHubHTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json deleted file mode 100644 index 1acd84f..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f177f1fc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 210.71 \nPeak output token throughput (tok/s): 430.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 411.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 145.79 \nMedian TTFT (ms): 110.27 \nP99 TTFT (ms): 399.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.46 \nMedian TPOT (ms): 22.67 \nP99 TPOT (ms): 36.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.37 \nMedian ITL (ms): 21.34 \nP99 ITL (ms): 84.18 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json deleted file mode 100644 index de4d720..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31b9a516-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 215.08 \nTotal input tokens: 146694 \nTotal generated tokens: 155647 \nRequest throughput (req/s): 3.35 \nOutput token throughput (tok/s): 723.67 \nPeak output token throughput (tok/s): 1208.00 \nPeak concurrent requests: 113.00 \nTotal Token throughput (tok/s): 1405.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 11329.86 \nMedian TTFT (ms): 14049.92 \nP99 TTFT (ms): 20450.77 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 40.85 \nMedian TPOT (ms): 39.82 \nP99 TPOT (ms): 83.14 \n---------------Inter-token Latency----------------\nMean ITL (ms): 40.32 \nMedian ITL (ms): 26.70 \nP99 ITL (ms): 338.67 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log deleted file mode 100644 index 96e14f0..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log +++ /dev/null @@ -1,1027 +0,0 @@ -WARNING 12-10 09:23:01 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=3017) INFO 12-10 09:23:01 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=3017) INFO 12-10 09:23:01 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 4096, 'gpu_memory_utilization': 0.86, 'max_num_seqs': 32} -(APIServer pid=3017) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=3017) INFO 12-10 09:23:09 [model.py:637] Resolved architecture: Qwen3ForCausalLM -(APIServer pid=3017) INFO 12-10 09:23:09 [model.py:1750] Using max model len 4096 -(APIServer pid=3017) INFO 12-10 09:23:09 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=3057) INFO 12-10 09:23:18 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:36297 backend=nccl -(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... -(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=3057) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-db5fc148-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.17 \nTotal input tokens: 38358 \nTotal generated tokens: 40015 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 217.27 \nPeak output token throughput (tok/s): 541.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 425.55 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.17 \nMedian TTFT (ms): 40.63 \nP99 TTFT (ms): 140.17 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 9.73 \nMedian TPOT (ms): 9.55 \nP99 TPOT (ms): 15.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 9.55 \nMedian ITL (ms): 9.45 \nP99 ITL (ms): 14.11 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json deleted file mode 100644 index 5e5dc50..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c07c837c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 189.59 \nTotal input tokens: 146694 \nTotal generated tokens: 153522 \nRequest throughput (req/s): 3.80 \nOutput token throughput (tok/s): 809.76 \nPeak output token throughput (tok/s): 1387.00 \nPeak concurrent requests: 40.00 \nTotal Token throughput (tok/s): 1583.51 \n---------------Time to First Token----------------\nMean TTFT (ms): 83.62 \nMedian TTFT (ms): 71.44 \nP99 TTFT (ms): 196.63 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.34 \nMedian TPOT (ms): 21.97 \nP99 TPOT (ms): 35.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.92 \nMedian ITL (ms): 19.31 \nP99 ITL (ms): 107.58 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log deleted file mode 100644 index fc8e21f..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log +++ /dev/null @@ -1,1026 +0,0 @@ -WARNING 12-10 09:44:20 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=3923) INFO 12-10 09:44:20 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=3923) INFO 12-10 09:44:20 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} -(APIServer pid=3923) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=3923) INFO 12-10 09:44:29 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM -(APIServer pid=3923) INFO 12-10 09:44:29 [model.py:1750] Using max model len 24576 -(APIServer pid=3923) INFO 12-10 09:44:29 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:37 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:38 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:49511 backend=nccl -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:38 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:38 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod -(EngineCore_DP0 pid=3963) WARNING 12-10 09:44:39 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod -(EngineCore_DP0 pid=3963) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bec359dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.18 \nTotal input tokens: 37841 \nTotal generated tokens: 38760 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 202.74 \nPeak output token throughput (tok/s): 418.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 400.67 \n---------------Time to First Token----------------\nMean TTFT (ms): 95.72 \nMedian TTFT (ms): 55.76 \nP99 TTFT (ms): 257.24 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.29 \nMedian TPOT (ms): 23.04 \nP99 TPOT (ms): 28.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.29 \nMedian ITL (ms): 22.01 \nP99 ITL (ms): 67.09 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json deleted file mode 100644 index 903dc61..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-acbcf111-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 199.30 \nTotal input tokens: 145810 \nTotal generated tokens: 151171 \nRequest throughput (req/s): 3.61 \nOutput token throughput (tok/s): 758.52 \nPeak output token throughput (tok/s): 1326.00 \nPeak concurrent requests: 52.00 \nTotal Token throughput (tok/s): 1490.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 134.75 \nMedian TTFT (ms): 130.98 \nP99 TTFT (ms): 409.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 36.74 \nMedian TPOT (ms): 35.30 \nP99 TPOT (ms): 77.70 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.30 \nMedian ITL (ms): 26.57 \nP99 ITL (ms): 192.53 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log deleted file mode 100644 index 1a11313..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log +++ /dev/null @@ -1,1023 +0,0 @@ -WARNING 12-10 10:12:27 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=4826) INFO 12-10 10:12:27 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=4826) INFO 12-10 10:12:27 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 31800, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=4826) INFO 12-10 10:12:36 [model.py:637] Resolved architecture: LlamaForCausalLM -(APIServer pid=4826) INFO 12-10 10:12:36 [model.py:1750] Using max model len 31800 -(APIServer pid=4826) INFO 12-10 10:12:36 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=4866) INFO 12-10 10:12:46 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=31800, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=4866) INFO 12-10 10:12:46 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:45209 backend=nccl -(EngineCore_DP0 pid=4866) INFO 12-10 10:12:46 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=4866) INFO 12-10 10:12:47 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... -(EngineCore_DP0 pid=4866) INFO 12-10 10:12:47 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=4866) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b23a48ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.20 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.78 \nPeak output token throughput (tok/s): 495.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 423.18 \n---------------Time to First Token----------------\nMean TTFT (ms): 60.23 \nMedian TTFT (ms): 37.85 \nP99 TTFT (ms): 146.84 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.21 \nMedian TPOT (ms): 11.26 \nP99 TPOT (ms): 18.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.00 \nMedian ITL (ms): 10.95 \nP99 ITL (ms): 19.93 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json deleted file mode 100644 index 4ea3aa8..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2c7096a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.29 \nTotal input tokens: 145540 \nTotal generated tokens: 151404 \nRequest throughput (req/s): 3.78 \nOutput token throughput (tok/s): 795.67 \nPeak output token throughput (tok/s): 1315.00 \nPeak concurrent requests: 36.00 \nTotal Token throughput (tok/s): 1560.51 \n---------------Time to First Token----------------\nMean TTFT (ms): 65.82 \nMedian TTFT (ms): 45.53 \nP99 TTFT (ms): 177.98 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 20.36 \nMedian TPOT (ms): 20.02 \nP99 TPOT (ms): 31.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 20.10 \nMedian ITL (ms): 18.54 \nP99 ITL (ms): 90.23 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_server.log deleted file mode 100644 index 7036a6f..0000000 --- a/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_server.log +++ /dev/null @@ -1,1022 +0,0 @@ -WARNING 12-10 08:45:12 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=1916) INFO 12-10 08:45:12 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=1916) INFO 12-10 08:45:12 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 16384, 'max_num_seqs': 32} -(APIServer pid=1916) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=1916) INFO 12-10 08:45:19 [model.py:637] Resolved architecture: GptOssForCausalLM -(APIServer pid=1916) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=1960) INFO 12-10 08:45:30 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:59799 backend=nccl -(EngineCore_DP0 pid=1960) INFO 12-10 08:45:30 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=1960) INFO 12-10 08:45:30 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... -(EngineCore_DP0 pid=1960) INFO 12-10 08:45:31 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] -(EngineCore_DP0 pid=1960) INFO 12-10 08:45:31 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=1960) INFO 12-10 08:45:31 [mxfp4.py:162] Using Marlin backend -(EngineCore_DP0 pid=1960) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96888f07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.82 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 211.17 \nPeak output token throughput (tok/s): 443.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 412.18 \n---------------Time to First Token----------------\nMean TTFT (ms): 64.41 \nMedian TTFT (ms): 50.82 \nP99 TTFT (ms): 144.55 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 21.14 \nMedian TPOT (ms): 20.98 \nP99 TPOT (ms): 23.59 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.12 \nMedian ITL (ms): 20.65 \nP99 ITL (ms): 29.23 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json deleted file mode 100644 index d623116..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bd32e896-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 194.55 \nTotal input tokens: 146694 \nTotal generated tokens: 155576 \nRequest throughput (req/s): 3.70 \nOutput token throughput (tok/s): 799.66 \nPeak output token throughput (tok/s): 1300.00 \nPeak concurrent requests: 43.00 \nTotal Token throughput (tok/s): 1553.66 \n---------------Time to First Token----------------\nMean TTFT (ms): 92.77 \nMedian TTFT (ms): 68.07 \nP99 TTFT (ms): 484.98 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.78 \nMedian TPOT (ms): 25.52 \nP99 TPOT (ms): 34.37 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.65 \nMedian ITL (ms): 23.39 \nP99 ITL (ms): 86.03 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log deleted file mode 100644 index 49fed97..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log +++ /dev/null @@ -1,1024 +0,0 @@ -WARNING 12-09 21:03:27 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=31169) INFO 12-09 21:03:27 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=31169) INFO 12-09 21:03:27 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 4096, 'gpu_memory_utilization': 0.86, 'max_num_seqs': 32} -(APIServer pid=31169) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=31169) INFO 12-09 21:03:34 [model.py:637] Resolved architecture: Qwen3ForCausalLM -(APIServer pid=31169) INFO 12-09 21:03:34 [model.py:1750] Using max model len 4096 -(APIServer pid=31169) INFO 12-09 21:03:34 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=31480) INFO 12-09 21:03:41 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=31480) INFO 12-09 21:03:42 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:34731 backend=nccl -(EngineCore_DP0 pid=31480) INFO 12-09 21:03:42 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=31480) INFO 12-09 21:03:42 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... -(EngineCore_DP0 pid=31480) INFO 12-09 21:03:43 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=31480) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4cdb5c8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 183.35 \nTotal input tokens: 38358 \nTotal generated tokens: 39718 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 216.63 \nPeak output token throughput (tok/s): 666.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 425.84 \n---------------Time to First Token----------------\nMean TTFT (ms): 37.48 \nMedian TTFT (ms): 30.33 \nP99 TTFT (ms): 77.17 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 7.36 \nMedian TPOT (ms): 7.11 \nP99 TPOT (ms): 11.45 \n---------------Inter-token Latency----------------\nMean ITL (ms): 7.26 \nMedian ITL (ms): 6.53 \nP99 ITL (ms): 11.05 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json deleted file mode 100644 index 0211464..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-81c0edc1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 186.20 \nTotal input tokens: 146694 \nTotal generated tokens: 153033 \nRequest throughput (req/s): 3.87 \nOutput token throughput (tok/s): 821.89 \nPeak output token throughput (tok/s): 1471.00 \nPeak concurrent requests: 30.00 \nTotal Token throughput (tok/s): 1609.74 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.36 \nMedian TTFT (ms): 42.14 \nP99 TTFT (ms): 82.58 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 13.89 \nMedian TPOT (ms): 13.91 \nP99 TPOT (ms): 17.75 \n---------------Inter-token Latency----------------\nMean ITL (ms): 13.74 \nMedian ITL (ms): 13.29 \nP99 ITL (ms): 38.23 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log deleted file mode 100644 index 29c36f8..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log +++ /dev/null @@ -1,1026 +0,0 @@ -WARNING 12-09 21:18:19 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=35981) INFO 12-09 21:18:19 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=35981) INFO 12-09 21:18:19 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} -(APIServer pid=35981) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=35981) INFO 12-09 21:18:25 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM -(APIServer pid=35981) INFO 12-09 21:18:25 [model.py:1750] Using max model len 24576 -(APIServer pid=35981) INFO 12-09 21:18:26 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:33 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:34 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:58697 backend=nccl -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:34 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:34 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod -(EngineCore_DP0 pid=36287) WARNING 12-09 21:18:35 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod -(EngineCore_DP0 pid=36287) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-04789567-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 188.87 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.95 \nOutput token throughput (tok/s): 205.96 \nPeak output token throughput (tok/s): 465.00 \nPeak concurrent requests: 10.00 \nTotal Token throughput (tok/s): 406.32 \n---------------Time to First Token----------------\nMean TTFT (ms): 52.16 \nMedian TTFT (ms): 44.25 \nP99 TTFT (ms): 112.79 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 17.68 \nMedian TPOT (ms): 17.62 \nP99 TPOT (ms): 19.35 \n---------------Inter-token Latency----------------\nMean ITL (ms): 17.64 \nMedian ITL (ms): 17.35 \nP99 ITL (ms): 23.66 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json deleted file mode 100644 index a8b55d3..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b098e8e8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 193.10 \nTotal input tokens: 145810 \nTotal generated tokens: 152091 \nRequest throughput (req/s): 3.73 \nOutput token throughput (tok/s): 787.62 \nPeak output token throughput (tok/s): 1351.00 \nPeak concurrent requests: 39.00 \nTotal Token throughput (tok/s): 1542.71 \n---------------Time to First Token----------------\nMean TTFT (ms): 57.32 \nMedian TTFT (ms): 49.20 \nP99 TTFT (ms): 126.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 20.27 \nMedian TPOT (ms): 20.07 \nP99 TPOT (ms): 27.95 \n---------------Inter-token Latency----------------\nMean ITL (ms): 20.15 \nMedian ITL (ms): 19.30 \nP99 ITL (ms): 62.76 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log deleted file mode 100644 index 8e6a030..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log +++ /dev/null @@ -1,1023 +0,0 @@ -WARNING 12-09 19:47:08 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=5841) INFO 12-09 19:47:08 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=5841) INFO 12-09 19:47:08 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 31800, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=5841) INFO 12-09 19:47:17 [model.py:637] Resolved architecture: LlamaForCausalLM -(APIServer pid=5841) INFO 12-09 19:47:17 [model.py:1750] Using max model len 31800 -(APIServer pid=5841) INFO 12-09 19:47:17 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=6154) INFO 12-09 19:47:26 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=31800, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=6154) INFO 12-09 19:47:26 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:35389 backend=nccl -(EngineCore_DP0 pid=6154) INFO 12-09 19:47:26 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=6154) INFO 12-09 19:47:27 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... -(EngineCore_DP0 pid=6154) INFO 12-09 19:47:27 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=6154) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31ba87dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 183.12 \nTotal input tokens: 38756 \nTotal generated tokens: 38869 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.27 \nPeak output token throughput (tok/s): 609.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 423.91 \n---------------Time to First Token----------------\nMean TTFT (ms): 40.22 \nMedian TTFT (ms): 36.83 \nP99 TTFT (ms): 74.88 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.07 \nMedian TPOT (ms): 8.23 \nP99 TPOT (ms): 11.78 \n---------------Inter-token Latency----------------\nMean ITL (ms): 8.04 \nMedian ITL (ms): 8.17 \nP99 ITL (ms): 12.73 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json deleted file mode 100644 index 7a9f6b1..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b94205ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 187.07 \nTotal input tokens: 145540 \nTotal generated tokens: 151811 \nRequest throughput (req/s): 3.85 \nOutput token throughput (tok/s): 811.50 \nPeak output token throughput (tok/s): 1549.00 \nPeak concurrent requests: 30.00 \nTotal Token throughput (tok/s): 1589.49 \n---------------Time to First Token----------------\nMean TTFT (ms): 41.02 \nMedian TTFT (ms): 36.91 \nP99 TTFT (ms): 79.84 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.71 \nMedian TPOT (ms): 14.74 \nP99 TPOT (ms): 17.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.60 \nMedian ITL (ms): 14.44 \nP99 ITL (ms): 32.49 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_server.log deleted file mode 100644 index 179a843..0000000 --- a/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_server.log +++ /dev/null @@ -1,1022 +0,0 @@ -WARNING 12-09 20:20:45 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=19409) INFO 12-09 20:20:45 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=19409) INFO 12-09 20:20:45 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 16384, 'max_num_seqs': 32} -(APIServer pid=19409) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=19409) INFO 12-09 20:20:53 [model.py:637] Resolved architecture: GptOssForCausalLM -(APIServer pid=19409) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=19724) INFO 12-09 20:21:02 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:56281 backend=nccl -(EngineCore_DP0 pid=19724) INFO 12-09 20:21:02 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... -(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] -(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [mxfp4.py:162] Using Marlin backend -(EngineCore_DP0 pid=19724) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0abaed61-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 192.06 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 209.81 \nPeak output token throughput (tok/s): 440.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 409.54 \n---------------Time to First Token----------------\nMean TTFT (ms): 51.40 \nMedian TTFT (ms): 47.89 \nP99 TTFT (ms): 93.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.06 \nMedian TPOT (ms): 23.00 \nP99 TPOT (ms): 24.12 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.10 \nMedian ITL (ms): 22.83 \nP99 ITL (ms): 28.65 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json deleted file mode 100644 index 96eddac..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-14719fc9-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 195.24 \nTotal input tokens: 146694 \nTotal generated tokens: 155585 \nRequest throughput (req/s): 3.69 \nOutput token throughput (tok/s): 796.89 \nPeak output token throughput (tok/s): 1321.00 \nPeak concurrent requests: 39.00 \nTotal Token throughput (tok/s): 1548.24 \n---------------Time to First Token----------------\nMean TTFT (ms): 53.18 \nMedian TTFT (ms): 49.55 \nP99 TTFT (ms): 101.04 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 24.41 \nMedian TPOT (ms): 24.26 \nP99 TPOT (ms): 28.40 \n---------------Inter-token Latency----------------\nMean ITL (ms): 24.37 \nMedian ITL (ms): 23.61 \nP99 ITL (ms): 56.70 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log deleted file mode 100644 index 64143d2..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log +++ /dev/null @@ -1,1032 +0,0 @@ -WARNING 12-09 19:04:10 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=14033) INFO 12-09 19:04:10 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=14033) INFO 12-09 19:04:10 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64} -(APIServer pid=14033) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=14033) INFO 12-09 19:04:16 [model.py:637] Resolved architecture: Qwen3ForCausalLM -(APIServer pid=14033) INFO 12-09 19:04:16 [model.py:1750] Using max model len 32768 -(APIServer pid=14033) INFO 12-09 19:04:17 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=14297) INFO 12-09 19:04:23 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:39463 backend=nccl -(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... -(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=14297) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5741d2ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 182.76 \nTotal input tokens: 38358 \nTotal generated tokens: 39611 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 216.74 \nPeak output token throughput (tok/s): 789.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 426.61 \n---------------Time to First Token----------------\nMean TTFT (ms): 30.25 \nMedian TTFT (ms): 23.16 \nP99 TTFT (ms): 57.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 5.38 \nMedian TPOT (ms): 5.32 \nP99 TPOT (ms): 7.38 \n---------------Inter-token Latency----------------\nMean ITL (ms): 5.36 \nMedian ITL (ms): 5.29 \nP99 ITL (ms): 7.43 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json deleted file mode 100644 index ab807da..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4730dd07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 183.59 \nTotal input tokens: 146694 \nTotal generated tokens: 152861 \nRequest throughput (req/s): 3.92 \nOutput token throughput (tok/s): 832.64 \nPeak output token throughput (tok/s): 1713.00 \nPeak concurrent requests: 25.00 \nTotal Token throughput (tok/s): 1631.69 \n---------------Time to First Token----------------\nMean TTFT (ms): 29.84 \nMedian TTFT (ms): 24.09 \nP99 TTFT (ms): 60.04 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.01 \nMedian TPOT (ms): 7.77 \nP99 TPOT (ms): 11.35 \n---------------Inter-token Latency----------------\nMean ITL (ms): 7.96 \nMedian ITL (ms): 7.28 \nP99 ITL (ms): 27.43 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log deleted file mode 100644 index 5575e22..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log +++ /dev/null @@ -1,1029 +0,0 @@ -WARNING 12-09 19:23:27 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=17193) INFO 12-09 19:23:27 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=17193) INFO 12-09 19:23:27 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} -(APIServer pid=17193) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=17193) INFO 12-09 19:23:33 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM -(APIServer pid=17193) INFO 12-09 19:23:33 [model.py:1750] Using max model len 24576 -(APIServer pid=17193) INFO 12-09 19:23:35 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:41 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:60659 backend=nccl -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:43 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:43 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=17457) INFO 12-09 19:23:43 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod -(EngineCore_DP0 pid=17457) WARNING 12-09 19:23:43 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod -(EngineCore_DP0 pid=17457) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d6b41380-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 185.59 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.97 \nOutput token throughput (tok/s): 209.60 \nPeak output token throughput (tok/s): 517.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 413.50 \n---------------Time to First Token----------------\nMean TTFT (ms): 39.73 \nMedian TTFT (ms): 32.58 \nP99 TTFT (ms): 92.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.35 \nMedian TPOT (ms): 11.23 \nP99 TPOT (ms): 13.06 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.32 \nMedian ITL (ms): 11.05 \nP99 ITL (ms): 13.41 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json deleted file mode 100644 index aa4f809..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-10085502-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 188.38 \nTotal input tokens: 145810 \nTotal generated tokens: 152175 \nRequest throughput (req/s): 3.82 \nOutput token throughput (tok/s): 807.79 \nPeak output token throughput (tok/s): 1510.00 \nPeak concurrent requests: 29.00 \nTotal Token throughput (tok/s): 1581.79 \n---------------Time to First Token----------------\nMean TTFT (ms): 38.75 \nMedian TTFT (ms): 31.18 \nP99 TTFT (ms): 92.33 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 12.77 \nMedian TPOT (ms): 12.43 \nP99 TPOT (ms): 17.06 \n---------------Inter-token Latency----------------\nMean ITL (ms): 12.71 \nMedian ITL (ms): 11.91 \nP99 ITL (ms): 36.70 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log deleted file mode 100644 index 2a23afe..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log +++ /dev/null @@ -1,1027 +0,0 @@ -WARNING 12-09 18:18:16 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=4406) INFO 12-09 18:18:16 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=4406) INFO 12-09 18:18:16 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=4406) INFO 12-09 18:18:23 [model.py:637] Resolved architecture: LlamaForCausalLM -(APIServer pid=4406) INFO 12-09 18:18:23 [model.py:1750] Using max model len 65536 -(APIServer pid=4406) INFO 12-09 18:18:24 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=4670) INFO 12-09 18:18:31 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=4670) INFO 12-09 18:18:31 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:34945 backend=nccl -(EngineCore_DP0 pid=4670) INFO 12-09 18:18:31 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=4670) INFO 12-09 18:18:32 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... -(EngineCore_DP0 pid=4670) INFO 12-09 18:18:32 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=4670) INFO 12-09 18:18:34 [weight_utils.py:487] Time spent downloading weights for meta-llama/Meta-Llama-3.1-8B-Instruct: 0.680210 seconds -(EngineCore_DP0 pid=4670) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7ca49d8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 182.22 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.99 \nOutput token throughput (tok/s): 215.09 \nPeak output token throughput (tok/s): 788.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 427.78 \n---------------Time to First Token----------------\nMean TTFT (ms): 26.17 \nMedian TTFT (ms): 21.35 \nP99 TTFT (ms): 52.39 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 5.08 \nMedian TPOT (ms): 5.07 \nP99 TPOT (ms): 7.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 5.10 \nMedian ITL (ms): 5.12 \nP99 ITL (ms): 7.77 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json deleted file mode 100644 index 81bccdf..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1904b29f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 183.65 \nTotal input tokens: 145540 \nTotal generated tokens: 151340 \nRequest throughput (req/s): 3.92 \nOutput token throughput (tok/s): 824.06 \nPeak output token throughput (tok/s): 1801.00 \nPeak concurrent requests: 25.00 \nTotal Token throughput (tok/s): 1616.54 \n---------------Time to First Token----------------\nMean TTFT (ms): 26.49 \nMedian TTFT (ms): 22.28 \nP99 TTFT (ms): 56.70 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.24 \nMedian TPOT (ms): 8.09 \nP99 TPOT (ms): 10.51 \n---------------Inter-token Latency----------------\nMean ITL (ms): 8.19 \nMedian ITL (ms): 7.72 \nP99 ITL (ms): 14.51 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_server.log deleted file mode 100644 index e415e3d..0000000 --- a/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_server.log +++ /dev/null @@ -1,1030 +0,0 @@ -WARNING 12-09 18:35:12 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=8738) INFO 12-09 18:35:12 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=8738) INFO 12-09 18:35:12 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=8738) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=8738) INFO 12-09 18:35:19 [model.py:637] Resolved architecture: GptOssForCausalLM -(APIServer pid=8738) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:35797 backend=nccl -(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... -(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] -(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [mxfp4.py:162] Using Marlin backend -(EngineCore_DP0 pid=9006) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-66e0ceca-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 187.88 \nTotal input tokens: 38358 \nTotal generated tokens: 40286 \nRequest throughput (req/s): 0.96 \nOutput token throughput (tok/s): 214.42 \nPeak output token throughput (tok/s): 460.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 418.59 \n---------------Time to First Token----------------\nMean TTFT (ms): 77.11 \nMedian TTFT (ms): 52.45 \nP99 TTFT (ms): 199.22 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 15.76 \nMedian TPOT (ms): 15.43 \nP99 TPOT (ms): 21.30 \n---------------Inter-token Latency----------------\nMean ITL (ms): 15.60 \nMedian ITL (ms): 14.96 \nP99 ITL (ms): 17.95 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json deleted file mode 100644 index ff2975f..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-702c15c4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.21 \nTotal input tokens: 146694 \nTotal generated tokens: 155632 \nRequest throughput (req/s): 3.79 \nOutput token throughput (tok/s): 818.19 \nPeak output token throughput (tok/s): 1520.00 \nPeak concurrent requests: 37.00 \nTotal Token throughput (tok/s): 1589.40 \n---------------Time to First Token----------------\nMean TTFT (ms): 76.15 \nMedian TTFT (ms): 45.58 \nP99 TTFT (ms): 247.34 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 19.54 \nMedian TPOT (ms): 18.83 \nP99 TPOT (ms): 35.16 \n---------------Inter-token Latency----------------\nMean ITL (ms): 19.12 \nMedian ITL (ms): 16.70 \nP99 ITL (ms): 114.66 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log deleted file mode 100644 index 8e23e99..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log +++ /dev/null @@ -1,1024 +0,0 @@ -WARNING 12-10 09:47:44 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=6553) INFO 12-10 09:47:44 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=6553) INFO 12-10 09:47:44 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64} -(APIServer pid=6553) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=6553) INFO 12-10 09:47:54 [model.py:637] Resolved architecture: Qwen3ForCausalLM -(APIServer pid=6553) INFO 12-10 09:47:54 [model.py:1750] Using max model len 32768 -(APIServer pid=6553) INFO 12-10 09:47:54 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=6815) INFO 12-10 09:48:03 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=6815) INFO 12-10 09:48:03 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:36973 backend=nccl -(EngineCore_DP0 pid=6815) INFO 12-10 09:48:03 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=6815) INFO 12-10 09:48:04 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... -(EngineCore_DP0 pid=6815) INFO 12-10 09:48:05 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=6815) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b80b1bb5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.14 \nTotal input tokens: 38358 \nTotal generated tokens: 39211 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.94 \nPeak output token throughput (tok/s): 560.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 421.26 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.12 \nMedian TTFT (ms): 41.56 \nP99 TTFT (ms): 82.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.89 \nMedian TPOT (ms): 8.73 \nP99 TPOT (ms): 13.87 \n---------------Inter-token Latency----------------\nMean ITL (ms): 8.70 \nMedian ITL (ms): 8.44 \nP99 ITL (ms): 11.78 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json deleted file mode 100644 index 6f2703c..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fb44348e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 186.80 \nTotal input tokens: 146694 \nTotal generated tokens: 153333 \nRequest throughput (req/s): 3.85 \nOutput token throughput (tok/s): 820.85 \nPeak output token throughput (tok/s): 1449.00 \nPeak concurrent requests: 31.00 \nTotal Token throughput (tok/s): 1606.16 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.68 \nMedian TTFT (ms): 36.77 \nP99 TTFT (ms): 89.21 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.46 \nMedian TPOT (ms): 14.20 \nP99 TPOT (ms): 19.40 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.28 \nMedian ITL (ms): 13.63 \nP99 ITL (ms): 44.44 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log deleted file mode 100644 index 8c135cc..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log +++ /dev/null @@ -1,1026 +0,0 @@ -WARNING 12-10 10:05:09 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=8737) INFO 12-10 10:05:09 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=8737) INFO 12-10 10:05:09 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} -(APIServer pid=8737) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=8737) INFO 12-10 10:05:19 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM -(APIServer pid=8737) INFO 12-10 10:05:19 [model.py:1750] Using max model len 24576 -(APIServer pid=8737) INFO 12-10 10:05:19 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:28 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:47671 backend=nccl -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:30 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:30 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=8999) INFO 12-10 10:05:30 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod -(EngineCore_DP0 pid=8999) WARNING 12-10 10:05:30 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod -(EngineCore_DP0 pid=8999) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4225f4cd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 187.19 \nTotal input tokens: 37841 \nTotal generated tokens: 38830 \nRequest throughput (req/s): 0.96 \nOutput token throughput (tok/s): 207.43 \nPeak output token throughput (tok/s): 490.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 409.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 45.84 \nMedian TTFT (ms): 37.87 \nP99 TTFT (ms): 101.24 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.35 \nMedian TPOT (ms): 14.27 \nP99 TPOT (ms): 16.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.28 \nMedian ITL (ms): 14.04 \nP99 ITL (ms): 16.81 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json deleted file mode 100644 index 5346632..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ca67e7d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.55 \nTotal input tokens: 145810 \nTotal generated tokens: 152171 \nRequest throughput (req/s): 3.78 \nOutput token throughput (tok/s): 798.60 \nPeak output token throughput (tok/s): 1492.00 \nPeak concurrent requests: 35.00 \nTotal Token throughput (tok/s): 1563.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.19 \nMedian TTFT (ms): 37.54 \nP99 TTFT (ms): 101.54 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 16.05 \nMedian TPOT (ms): 15.69 \nP99 TPOT (ms): 20.99 \n---------------Inter-token Latency----------------\nMean ITL (ms): 15.95 \nMedian ITL (ms): 15.02 \nP99 ITL (ms): 41.81 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log deleted file mode 100644 index 354c6bc..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log +++ /dev/null @@ -1,1022 +0,0 @@ -WARNING 12-10 09:24:11 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=2740) INFO 12-10 09:24:11 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=2740) INFO 12-10 09:24:11 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=2740) INFO 12-10 09:24:21 [model.py:637] Resolved architecture: LlamaForCausalLM -(APIServer pid=2740) INFO 12-10 09:24:21 [model.py:1750] Using max model len 65536 -(APIServer pid=2740) INFO 12-10 09:24:21 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=3002) INFO 12-10 09:24:30 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=3002) INFO 12-10 09:24:30 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:46573 backend=nccl -(EngineCore_DP0 pid=3002) INFO 12-10 09:24:30 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=3002) INFO 12-10 09:24:31 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... -(EngineCore_DP0 pid=3002) INFO 12-10 09:24:32 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=3002) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-85d6e157-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 183.05 \nTotal input tokens: 38756 \nTotal generated tokens: 38779 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 211.85 \nPeak output token throughput (tok/s): 676.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 423.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 36.19 \nMedian TTFT (ms): 30.20 \nP99 TTFT (ms): 83.37 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 7.40 \nMedian TPOT (ms): 7.32 \nP99 TPOT (ms): 10.60 \n---------------Inter-token Latency----------------\nMean ITL (ms): 7.34 \nMedian ITL (ms): 7.07 \nP99 ITL (ms): 10.69 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json deleted file mode 100644 index f8b7077..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-584a872e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 185.93 \nTotal input tokens: 145540 \nTotal generated tokens: 151690 \nRequest throughput (req/s): 3.87 \nOutput token throughput (tok/s): 815.86 \nPeak output token throughput (tok/s): 1630.00 \nPeak concurrent requests: 28.00 \nTotal Token throughput (tok/s): 1598.63 \n---------------Time to First Token----------------\nMean TTFT (ms): 38.36 \nMedian TTFT (ms): 31.91 \nP99 TTFT (ms): 78.67 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 12.67 \nMedian TPOT (ms): 12.82 \nP99 TPOT (ms): 16.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 12.51 \nMedian ITL (ms): 12.62 \nP99 ITL (ms): 31.79 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_server.log deleted file mode 100644 index fb0eaef..0000000 --- a/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_server.log +++ /dev/null @@ -1,1023 +0,0 @@ -WARNING 12-10 10:51:19 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=15750) INFO 12-10 10:51:19 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=15750) INFO 12-10 10:51:19 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 24000, 'max_num_seqs': 64} -(APIServer pid=15750) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=15750) INFO 12-10 10:51:29 [model.py:637] Resolved architecture: GptOssForCausalLM -(APIServer pid=15750) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=16016) INFO 12-10 10:51:40 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:59277 backend=nccl -(EngineCore_DP0 pid=16016) INFO 12-10 10:51:40 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... -(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] -(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [mxfp4.py:162] Using Marlin backend -(EngineCore_DP0 pid=16016) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-39a9e102-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 197.48 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 204.06 \nPeak output token throughput (tok/s): 372.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 398.30 \n---------------Time to First Token----------------\nMean TTFT (ms): 78.01 \nMedian TTFT (ms): 70.55 \nP99 TTFT (ms): 149.81 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 34.15 \nMedian TPOT (ms): 34.06 \nP99 TPOT (ms): 36.67 \n---------------Inter-token Latency----------------\nMean ITL (ms): 34.16 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 58.95 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json deleted file mode 100644 index 1b3d37f..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0e099965-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 205.53 \nTotal input tokens: 146694 \nTotal generated tokens: 155647 \nRequest throughput (req/s): 3.50 \nOutput token throughput (tok/s): 757.31 \nPeak output token throughput (tok/s): 1260.00 \nPeak concurrent requests: 59.00 \nTotal Token throughput (tok/s): 1471.06 \n---------------Time to First Token----------------\nMean TTFT (ms): 85.03 \nMedian TTFT (ms): 76.62 \nP99 TTFT (ms): 176.99 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.66 \nMedian TPOT (ms): 41.32 \nP99 TPOT (ms): 52.93 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.47 \nMedian ITL (ms): 39.19 \nP99 ITL (ms): 111.74 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log deleted file mode 100644 index d2caafb..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log +++ /dev/null @@ -1,1025 +0,0 @@ -WARNING 12-10 19:52:40 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=4723) INFO 12-10 19:52:40 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=4723) INFO 12-10 19:52:40 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64} -(APIServer pid=4723) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=4723) INFO 12-10 19:52:45 [model.py:637] Resolved architecture: Qwen3ForCausalLM -(APIServer pid=4723) INFO 12-10 19:52:45 [model.py:1750] Using max model len 32768 -(APIServer pid=4723) INFO 12-10 19:52:45 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=4827) INFO 12-10 19:52:49 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=4827) INFO 12-10 19:52:50 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:34143 backend=nccl -(EngineCore_DP0 pid=4827) INFO 12-10 19:52:50 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=4827) INFO 12-10 19:52:50 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... -(EngineCore_DP0 pid=4827) INFO 12-10 19:52:51 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=4827) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-341eb0ef-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.76 \nTotal input tokens: 38358 \nTotal generated tokens: 40087 \nRequest throughput (req/s): 0.97 \nOutput token throughput (tok/s): 216.97 \nPeak output token throughput (tok/s): 453.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 424.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 40.37 \nMedian TTFT (ms): 37.70 \nP99 TTFT (ms): 70.13 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.50 \nMedian TPOT (ms): 11.56 \nP99 TPOT (ms): 16.13 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.24 \nMedian ITL (ms): 11.23 \nP99 ITL (ms): 19.54 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json deleted file mode 100644 index be18451..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-34321bd4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 192.26 \nTotal input tokens: 146694 \nTotal generated tokens: 153536 \nRequest throughput (req/s): 3.74 \nOutput token throughput (tok/s): 798.60 \nPeak output token throughput (tok/s): 1194.00 \nPeak concurrent requests: 46.00 \nTotal Token throughput (tok/s): 1561.61 \n---------------Time to First Token----------------\nMean TTFT (ms): 54.49 \nMedian TTFT (ms): 52.17 \nP99 TTFT (ms): 97.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.72 \nMedian TPOT (ms): 25.55 \nP99 TPOT (ms): 31.80 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.55 \nMedian ITL (ms): 24.86 \nP99 ITL (ms): 53.01 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log deleted file mode 100644 index 54d348e..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log +++ /dev/null @@ -1,1025 +0,0 @@ -WARNING 12-10 20:10:10 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=5918) INFO 12-10 20:10:10 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=5918) INFO 12-10 20:10:10 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} -(APIServer pid=5918) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=5918) INFO 12-10 20:10:17 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM -(APIServer pid=5918) INFO 12-10 20:10:17 [model.py:1750] Using max model len 24576 -(APIServer pid=5918) INFO 12-10 20:10:18 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:23 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:23 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:55455 backend=nccl -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:23 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod -(EngineCore_DP0 pid=6022) WARNING 12-10 20:10:24 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod -(EngineCore_DP0 pid=6022) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f41559c3-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.60 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.88 \nPeak output token throughput (tok/s): 395.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 392.34 \n---------------Time to First Token----------------\nMean TTFT (ms): 63.33 \nMedian TTFT (ms): 59.42 \nP99 TTFT (ms): 123.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.79 \nMedian TPOT (ms): 30.76 \nP99 TPOT (ms): 32.51 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.78 \nMedian ITL (ms): 30.39 \nP99 ITL (ms): 42.55 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json deleted file mode 100644 index dd7925e..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f456f8a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 719 \nFailed requests: 1 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.29 \nTotal input tokens: 145785 \nTotal generated tokens: 151865 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 743.36 \nPeak output token throughput (tok/s): 1197.00 \nPeak concurrent requests: 52.00 \nTotal Token throughput (tok/s): 1456.96 \n---------------Time to First Token----------------\nMean TTFT (ms): 68.23 \nMedian TTFT (ms): 63.70 \nP99 TTFT (ms): 136.63 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 34.84 \nMedian TPOT (ms): 34.66 \nP99 TPOT (ms): 41.64 \n---------------Inter-token Latency----------------\nMean ITL (ms): 34.70 \nMedian ITL (ms): 33.44 \nP99 ITL (ms): 79.44 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log deleted file mode 100644 index c84c39b..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log +++ /dev/null @@ -1,1021 +0,0 @@ -WARNING 12-10 19:17:54 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=1907) INFO 12-10 19:17:54 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=1907) INFO 12-10 19:17:54 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=1907) INFO 12-10 19:18:00 [model.py:637] Resolved architecture: LlamaForCausalLM -(APIServer pid=1907) INFO 12-10 19:18:00 [model.py:1750] Using max model len 65536 -(APIServer pid=1907) INFO 12-10 19:18:00 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. -(EngineCore_DP0 pid=2011) INFO 12-10 19:18:05 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=2011) INFO 12-10 19:18:06 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:45173 backend=nccl -(EngineCore_DP0 pid=2011) INFO 12-10 19:18:06 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=2011) INFO 12-10 19:18:06 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... -(EngineCore_DP0 pid=2011) INFO 12-10 19:18:07 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] -(EngineCore_DP0 pid=2011) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-68841ca7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 185.44 \nTotal input tokens: 38756 \nTotal generated tokens: 38777 \nRequest throughput (req/s): 0.97 \nOutput token throughput (tok/s): 209.11 \nPeak output token throughput (tok/s): 437.00 \nPeak concurrent requests: 10.00 \nTotal Token throughput (tok/s): 418.10 \n---------------Time to First Token----------------\nMean TTFT (ms): 42.77 \nMedian TTFT (ms): 40.02 \nP99 TTFT (ms): 73.37 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.89 \nMedian TPOT (ms): 14.86 \nP99 TPOT (ms): 21.46 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.57 \nMedian ITL (ms): 14.54 \nP99 ITL (ms): 24.86 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json deleted file mode 100644 index 0c20aeb..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "success": true, - "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96562708-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 194.32 \nTotal input tokens: 145540 \nTotal generated tokens: 151349 \nRequest throughput (req/s): 3.71 \nOutput token throughput (tok/s): 778.86 \nPeak output token throughput (tok/s): 1218.00 \nPeak concurrent requests: 42.00 \nTotal Token throughput (tok/s): 1527.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 51.01 \nMedian TTFT (ms): 49.88 \nP99 TTFT (ms): 83.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 26.15 \nMedian TPOT (ms): 26.20 \nP99 TPOT (ms): 28.93 \n---------------Inter-token Latency----------------\nMean ITL (ms): 26.04 \nMedian ITL (ms): 25.85 \nP99 ITL (ms): 42.96 \n==================================================\n" -} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_server.log deleted file mode 100644 index 5ee2883..0000000 --- a/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_server.log +++ /dev/null @@ -1,1022 +0,0 @@ -WARNING 12-10 19:34:31 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. -(APIServer pid=3207) INFO 12-10 19:34:31 [api_server.py:1772] vLLM API server version 0.12.0 -(APIServer pid=3207) INFO 12-10 19:34:31 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} -(APIServer pid=3207) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. -(APIServer pid=3207) INFO 12-10 19:34:38 [model.py:637] Resolved architecture: GptOssForCausalLM -(APIServer pid=3207) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} -(EngineCore_DP0 pid=3315) INFO 12-10 19:34:48 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:45235 backend=nccl -(EngineCore_DP0 pid=3315) INFO 12-10 19:34:48 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 -(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... -(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] -(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [layer.py:379] Enabled separate cuda stream for MoE shared_experts -(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [mxfp4.py:162] Using Marlin backend -(EngineCore_DP0 pid=3315) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00 ... (no positional model arg) - cmd = ["vllm", "serve"] + srv_args + ["--host", HOST, "--port", str(PORT)] - - ids_cmd = " ".join(cmd) - log(f"CMD: {ids_cmd}") - - proc = subprocess.Popen(cmd, stdout=srv_log, stderr=srv_log, env=env) - - try: - if not wait_for_server(f"http://{HOST}:{PORT}", proc): return - - for qps in QPS_SWEEP: - out_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{qps}_latency.json" - if out_file.exists(): continue - - log(f"BENCH QPS={qps}...") - bench_cmd = [ - "vllm", "bench", "serve", - "--model", model, - "--base-url", f"http://{HOST}:{PORT}", - "--request-rate", str(qps), - "--num-prompts", str(int(max(10, SRV_DURATION * qps))), - "--trust-remote-code" - ] - - if dataset_path: bench_cmd.extend(["--dataset-name", "sharegpt", "--dataset-path", dataset_path]) - else: bench_cmd.extend(["--dataset-name", "random", "--random-input-len", "1024", "--random-output-len", "512"]) - - res = subprocess.run(bench_cmd, capture_output=True, text=True, env=env) - with open(out_file, "w") as f: - f.write(json.dumps({"success": res.returncode==0, "raw_output": res.stdout}, indent=2)) - - except Exception as e: log(f"CRASH: {e}") - finally: - proc.terminate() - force_gpu_cleanup() - -def print_summary(tps): - print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}") - print("-" * 105) - - for m in MODELS_TO_RUN: - msafe = m.replace("/", "_") - for tp in tps: - if tp not in MODEL_TABLE[m]["valid_tp"]: continue - - try: - tdata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_throughput.json").read_text()) - tok_s = f"{tdata.get('tokens_per_second', 0):.1f}" - except: tok_s = "N/A" - - first_row = True - for q in QPS_SWEEP: - try: - ldata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_qps{q}_latency.json").read_text()) - raw = ldata["raw_output"] - ttft = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw).group(1) - tpot = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw).group(1) - except: ttft, tpot = "-", "-" - - name_cell = m.split('/')[-1] if (first_row and q == QPS_SWEEP[0]) else "" - - print(f"{name_cell:<40} | {tp:<2} | {tok_s:<8} | {q:<4} | {ttft:<6} | {tpot:<6}") - first_row = False - print("-" * 105) - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--tp", type=int, nargs="+", default=[1, 2]) - args = parser.parse_args() - - gpu_count = get_gpu_count() - log(f"Detected {gpu_count} GPU(s)") - - valid_tp_args = [t for t in args.tp if t <= gpu_count] - if not valid_tp_args: - log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.") - sys.exit(0) - - force_gpu_cleanup() - for tp in valid_tp_args: - for m in MODELS_TO_RUN: - run_throughput(m, tp) - run_latency(m, tp) - print_summary(valid_tp_args)