x
This commit is contained in:
+95
@@ -0,0 +1,95 @@
|
||||
WARNING 12-10 09:53:36 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:36 [api_server.py:1772] vLLM API server version 0.12.0
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:36 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'host': '127.0.0.1', 'model': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'trust_remote_code': True, 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64}
|
||||
[0;36m(APIServer pid=4288)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:47 [model.py:637] Resolved architecture: LlamaForCausalLM
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:47 [model.py:1750] Using max model len 65536
|
||||
[0;36m(APIServer pid=4288)[0;0m INFO 12-10 09:53:47 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
[0;36m(APIServer pid=4288)[0;0m Traceback (most recent call last):
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 409, in hf_raise_for_status
|
||||
[0;36m(APIServer pid=4288)[0;0m response.raise_for_status()
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/requests/models.py", line 1026, in raise_for_status
|
||||
[0;36m(APIServer pid=4288)[0;0m raise HTTPError(http_error_msg, response=self)
|
||||
[0;36m(APIServer pid=4288)[0;0m requests.exceptions.HTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block
|
||||
[0;36m(APIServer pid=4288)[0;0m
|
||||
[0;36m(APIServer pid=4288)[0;0m The above exception was the direct cause of the following exception:
|
||||
[0;36m(APIServer pid=4288)[0;0m
|
||||
[0;36m(APIServer pid=4288)[0;0m Traceback (most recent call last):
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/bin/vllm", line 7, in <module>
|
||||
[0;36m(APIServer pid=4288)[0;0m sys.exit(main())
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
|
||||
[0;36m(APIServer pid=4288)[0;0m args.dispatch_function(args)
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
|
||||
[0;36m(APIServer pid=4288)[0;0m uvloop.run(run_server(args))
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run
|
||||
[0;36m(APIServer pid=4288)[0;0m return __asyncio.run(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/asyncio/runners.py", line 195, in run
|
||||
[0;36m(APIServer pid=4288)[0;0m return runner.run(main)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/asyncio/runners.py", line 118, in run
|
||||
[0;36m(APIServer pid=4288)[0;0m return self._loop.run_until_complete(task)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper
|
||||
[0;36m(APIServer pid=4288)[0;0m return await main
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1819, in run_server
|
||||
[0;36m(APIServer pid=4288)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1838, in run_server_worker
|
||||
[0;36m(APIServer pid=4288)[0;0m async with build_async_engine_client(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
|
||||
[0;36m(APIServer pid=4288)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 183, in build_async_engine_client
|
||||
[0;36m(APIServer pid=4288)[0;0m async with build_async_engine_client_from_engine_args(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
|
||||
[0;36m(APIServer pid=4288)[0;0m return await anext(self.gen)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 224, in build_async_engine_client_from_engine_args
|
||||
[0;36m(APIServer pid=4288)[0;0m async_llm = AsyncLLM.from_vllm_config(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 223, in from_vllm_config
|
||||
[0;36m(APIServer pid=4288)[0;0m return cls(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 114, in __init__
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = init_tokenizer_from_config(self.model_config)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 227, in init_tokenizer_from_config
|
||||
[0;36m(APIServer pid=4288)[0;0m return get_tokenizer(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 191, in get_tokenizer
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = TokenizerRegistry.get_tokenizer(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 86, in get_tokenizer
|
||||
[0;36m(APIServer pid=4288)[0;0m return item.from_pretrained(*args, **kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/hf.py", line 84, in from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = AutoTokenizer.from_pretrained(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 1156, in from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2113, in from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m return cls._from_pretrained(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2395, in _from_pretrained
|
||||
[0;36m(APIServer pid=4288)[0;0m tokenizer = cls._patch_mistral_regex(
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2438, in _patch_mistral_regex
|
||||
[0;36m(APIServer pid=4288)[0;0m if _is_local or is_base_mistral(pretrained_model_name_or_path):
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2432, in is_base_mistral
|
||||
[0;36m(APIServer pid=4288)[0;0m model = model_info(model_id)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
|
||||
[0;36m(APIServer pid=4288)[0;0m return fn(*args, **kwargs)
|
||||
[0;36m(APIServer pid=4288)[0;0m ^^^^^^^^^^^^^^^^^^^
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/hf_api.py", line 2638, in model_info
|
||||
[0;36m(APIServer pid=4288)[0;0m hf_raise_for_status(r)
|
||||
[0;36m(APIServer pid=4288)[0;0m File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 482, in hf_raise_for_status
|
||||
[0;36m(APIServer pid=4288)[0;0m raise _format(HfHubHTTPError, str(e), response) from e
|
||||
[0;36m(APIServer pid=4288)[0;0m huggingface_hub.errors.HfHubHTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73dcacabf4c0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f177f1fc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 210.71 \nPeak output token throughput (tok/s): 430.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 411.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 145.79 \nMedian TTFT (ms): 110.27 \nP99 TTFT (ms): 399.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.46 \nMedian TPOT (ms): 22.67 \nP99 TPOT (ms): 36.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.37 \nMedian ITL (ms): 21.34 \nP99 ITL (ms): 84.18 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cabb891f240>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31b9a516-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 215.08 \nTotal input tokens: 146694 \nTotal generated tokens: 155647 \nRequest throughput (req/s): 3.35 \nOutput token throughput (tok/s): 723.67 \nPeak output token throughput (tok/s): 1208.00 \nPeak concurrent requests: 113.00 \nTotal Token throughput (tok/s): 1405.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 11329.86 \nMedian TTFT (ms): 14049.92 \nP99 TTFT (ms): 20450.77 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 40.85 \nMedian TPOT (ms): 39.82 \nP99 TPOT (ms): 83.14 \n---------------Inter-token Latency----------------\nMean ITL (ms): 40.32 \nMedian ITL (ms): 26.70 \nP99 ITL (ms): 338.67 \n==================================================\n"
|
||||
}
|
||||
+1027
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 1446.3949257107452,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 0.6913741069083253,
|
||||
"tokens_per_second": 512.5391321707765
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7efdf61eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-db5fc148-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.17 \nTotal input tokens: 38358 \nTotal generated tokens: 40015 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 217.27 \nPeak output token throughput (tok/s): 541.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 425.55 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.17 \nMedian TTFT (ms): 40.63 \nP99 TTFT (ms): 140.17 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 9.73 \nMedian TPOT (ms): 9.55 \nP99 TPOT (ms): 15.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 9.55 \nMedian ITL (ms): 9.45 \nP99 ITL (ms): 14.11 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7aa491426fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c07c837c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 189.59 \nTotal input tokens: 146694 \nTotal generated tokens: 153522 \nRequest throughput (req/s): 3.80 \nOutput token throughput (tok/s): 809.76 \nPeak output token throughput (tok/s): 1387.00 \nPeak concurrent requests: 40.00 \nTotal Token throughput (tok/s): 1583.51 \n---------------Time to First Token----------------\nMean TTFT (ms): 83.62 \nMedian TTFT (ms): 71.44 \nP99 TTFT (ms): 196.63 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.34 \nMedian TPOT (ms): 21.97 \nP99 TPOT (ms): 35.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.92 \nMedian ITL (ms): 19.31 \nP99 ITL (ms): 107.58 \n==================================================\n"
|
||||
}
|
||||
+1026
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 306.69494965299964,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 741334,
|
||||
"requests_per_second": 3.2605688523121055,
|
||||
"tokens_per_second": 2417.170549559942
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x78113ee3efc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bec359dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.18 \nTotal input tokens: 37841 \nTotal generated tokens: 38760 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 202.74 \nPeak output token throughput (tok/s): 418.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 400.67 \n---------------Time to First Token----------------\nMean TTFT (ms): 95.72 \nMedian TTFT (ms): 55.76 \nP99 TTFT (ms): 257.24 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.29 \nMedian TPOT (ms): 23.04 \nP99 TPOT (ms): 28.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.29 \nMedian ITL (ms): 22.01 \nP99 ITL (ms): 67.09 \n==================================================\n"
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7eff14b2efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-acbcf111-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 199.30 \nTotal input tokens: 145810 \nTotal generated tokens: 151171 \nRequest throughput (req/s): 3.61 \nOutput token throughput (tok/s): 758.52 \nPeak output token throughput (tok/s): 1326.00 \nPeak concurrent requests: 52.00 \nTotal Token throughput (tok/s): 1490.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 134.75 \nMedian TTFT (ms): 130.98 \nP99 TTFT (ms): 409.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 36.74 \nMedian TPOT (ms): 35.30 \nP99 TPOT (ms): 77.70 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.30 \nMedian ITL (ms): 26.57 \nP99 ITL (ms): 192.53 \n==================================================\n"
|
||||
}
|
||||
+1023
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 402.3294672546908,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 736330,
|
||||
"requests_per_second": 2.4855251265176648,
|
||||
"tokens_per_second": 1830.1667164087519
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7edefddeafc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b23a48ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.20 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.78 \nPeak output token throughput (tok/s): 495.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 423.18 \n---------------Time to First Token----------------\nMean TTFT (ms): 60.23 \nMedian TTFT (ms): 37.85 \nP99 TTFT (ms): 146.84 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.21 \nMedian TPOT (ms): 11.26 \nP99 TPOT (ms): 18.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.00 \nMedian ITL (ms): 10.95 \nP99 ITL (ms): 19.93 \n==================================================\n"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"success": true,
|
||||
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7400f615afc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2c7096a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.29 \nTotal input tokens: 145540 \nTotal generated tokens: 151404 \nRequest throughput (req/s): 3.78 \nOutput token throughput (tok/s): 795.67 \nPeak output token throughput (tok/s): 1315.00 \nPeak concurrent requests: 36.00 \nTotal Token throughput (tok/s): 1560.51 \n---------------Time to First Token----------------\nMean TTFT (ms): 65.82 \nMedian TTFT (ms): 45.53 \nP99 TTFT (ms): 177.98 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 20.36 \nMedian TPOT (ms): 20.02 \nP99 TPOT (ms): 31.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 20.10 \nMedian ITL (ms): 18.54 \nP99 ITL (ms): 90.23 \n==================================================\n"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"elapsed_time": 357.56243500020355,
|
||||
"num_requests": 1000,
|
||||
"total_num_tokens": 738792,
|
||||
"requests_per_second": 2.7967143696161223,
|
||||
"tokens_per_second": 2066.190202557434
|
||||
}
|
||||
Reference in New Issue
Block a user