x

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x71aab4572fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96888f07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  190.82    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         211.17    \nPeak output token throughput (tok/s):    443.00    \nPeak concurrent requests:                11.00     \nTotal Token throughput (tok/s):          412.18    \n---------------Time to First Token----------------\nMean TTFT (ms):                          64.41     \nMedian TTFT (ms):                        50.82     \nP99 TTFT (ms):                           144.55    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          21.14     \nMedian TPOT (ms):                        20.98     \nP99 TPOT (ms):                           23.59     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           21.12     \nMedian ITL (ms):                         20.65     \nP99 ITL (ms):                            29.23     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x76b9330fafc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bd32e896-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  194.55    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155576    \nRequest throughput (req/s):              3.70      \nOutput token throughput (tok/s):         799.66    \nPeak output token throughput (tok/s):    1300.00   \nPeak concurrent requests:                43.00     \nTotal Token throughput (tok/s):          1553.66   \n---------------Time to First Token----------------\nMean TTFT (ms):                          92.77     \nMedian TTFT (ms):                        68.07     \nP99 TTFT (ms):                           484.98    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          25.78     \nMedian TPOT (ms):                        25.52     \nP99 TPOT (ms):                           34.37     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           25.65     \nMedian ITL (ms):                         23.39     \nP99 ITL (ms):                            86.03     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 889.9994421191514,
+    "num_requests": 1000,
+    "total_num_tokens": 741334,
+    "requests_per_second": 1.1235962099245025,
+    "tokens_per_second": 832.9600726881711
+}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7236816eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4cdb5c8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.35    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39718     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         216.63    \nPeak output token throughput (tok/s):    666.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          425.84    \n---------------Time to First Token----------------\nMean TTFT (ms):                          37.48     \nMedian TTFT (ms):                        30.33     \nP99 TTFT (ms):                           77.17     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          7.36      \nMedian TPOT (ms):                        7.11      \nP99 TPOT (ms):                           11.45     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.26      \nMedian ITL (ms):                         6.53      \nP99 ITL (ms):                            11.05     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73a294576fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-81c0edc1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  186.20    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153033    \nRequest throughput (req/s):              3.87      \nOutput token throughput (tok/s):         821.89    \nPeak output token throughput (tok/s):    1471.00   \nPeak concurrent requests:                30.00     \nTotal Token throughput (tok/s):          1609.74   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.36     \nMedian TTFT (ms):                        42.14     \nP99 TTFT (ms):                           82.58     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          13.89     \nMedian TPOT (ms):                        13.91     \nP99 TPOT (ms):                           17.75     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           13.74     \nMedian ITL (ms):                         13.29     \nP99 ITL (ms):                            38.23     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 215.380199868232,
+    "num_requests": 1000,
+    "total_num_tokens": 741334,
+    "requests_per_second": 4.642952326220295,
+    "tokens_per_second": 3441.9784198061966
+}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x76db7ad72fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-04789567-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  188.87    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.95      \nOutput token throughput (tok/s):         205.96    \nPeak output token throughput (tok/s):    465.00    \nPeak concurrent requests:                10.00     \nTotal Token throughput (tok/s):          406.32    \n---------------Time to First Token----------------\nMean TTFT (ms):                          52.16     \nMedian TTFT (ms):                        44.25     \nP99 TTFT (ms):                           112.79    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          17.68     \nMedian TPOT (ms):                        17.62     \nP99 TPOT (ms):                           19.35     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           17.64     \nMedian ITL (ms):                         17.35     \nP99 ITL (ms):                            23.66     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7b8c3837efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b098e8e8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  193.10    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152091    \nRequest throughput (req/s):              3.73      \nOutput token throughput (tok/s):         787.62    \nPeak output token throughput (tok/s):    1351.00   \nPeak concurrent requests:                39.00     \nTotal Token throughput (tok/s):          1542.71   \n---------------Time to First Token----------------\nMean TTFT (ms):                          57.32     \nMedian TTFT (ms):                        49.20     \nP99 TTFT (ms):                           126.64    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          20.27     \nMedian TPOT (ms):                        20.07     \nP99 TPOT (ms):                           27.95     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           20.15     \nMedian ITL (ms):                         19.30     \nP99 ITL (ms):                            62.76     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 252.42282237578183,
+    "num_requests": 1000,
+    "total_num_tokens": 736330,
+    "requests_per_second": 3.9616069204364575,
+    "tokens_per_second": 2917.0500237249767
+}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7304080eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31ba87dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.12    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38869     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.27    \nPeak output token throughput (tok/s):    609.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          423.91    \n---------------Time to First Token----------------\nMean TTFT (ms):                          40.22     \nMedian TTFT (ms):                        36.83     \nP99 TTFT (ms):                           74.88     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.07      \nMedian TPOT (ms):                        8.23      \nP99 TPOT (ms):                           11.78     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.04      \nMedian ITL (ms):                         8.17      \nP99 ITL (ms):                            12.73     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cdc7797efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b94205ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  187.07    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151811    \nRequest throughput (req/s):              3.85      \nOutput token throughput (tok/s):         811.50    \nPeak output token throughput (tok/s):    1549.00   \nPeak concurrent requests:                30.00     \nTotal Token throughput (tok/s):          1589.49   \n---------------Time to First Token----------------\nMean TTFT (ms):                          41.02     \nMedian TTFT (ms):                        36.91     \nP99 TTFT (ms):                           79.84     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.71     \nMedian TPOT (ms):                        14.74     \nP99 TPOT (ms):                           17.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.60     \nMedian ITL (ms):                         14.44     \nP99 ITL (ms):                            32.49     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 275.10345687624067,
+    "num_requests": 1000,
+    "total_num_tokens": 738792,
+    "requests_per_second": 3.6349961260205634,
+    "tokens_per_second": 2685.5060579349843
+}