x

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f90e18fd9e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0abaed61-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  192.06    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         209.81    \nPeak output token throughput (tok/s):    440.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          409.54    \n---------------Time to First Token----------------\nMean TTFT (ms):                          51.40     \nMedian TTFT (ms):                        47.89     \nP99 TTFT (ms):                           93.19     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.06     \nMedian TPOT (ms):                        23.00     \nP99 TPOT (ms):                           24.12     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.10     \nMedian ITL (ms):                         22.83     \nP99 ITL (ms):                            28.65     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f205d7019e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-14719fc9-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  195.24    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155585    \nRequest throughput (req/s):              3.69      \nOutput token throughput (tok/s):         796.89    \nPeak output token throughput (tok/s):    1321.00   \nPeak concurrent requests:                39.00     \nTotal Token throughput (tok/s):          1548.24   \n---------------Time to First Token----------------\nMean TTFT (ms):                          53.18     \nMedian TTFT (ms):                        49.55     \nP99 TTFT (ms):                           101.04    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          24.41     \nMedian TPOT (ms):                        24.26     \nP99 TPOT (ms):                           28.40     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           24.37     \nMedian ITL (ms):                         23.61     \nP99 ITL (ms):                            56.70     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 228.03156735002995,
+    "num_requests": 1000,
+    "total_num_tokens": 741334,
+    "requests_per_second": 4.385357745074801,
+    "tokens_per_second": 3251.0147985872827
+}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fcbbe1e19e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5741d2ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  182.76    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39611     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         216.74    \nPeak output token throughput (tok/s):    789.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          426.61    \n---------------Time to First Token----------------\nMean TTFT (ms):                          30.25     \nMedian TTFT (ms):                        23.16     \nP99 TTFT (ms):                           57.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          5.38      \nMedian TPOT (ms):                        5.32      \nP99 TPOT (ms):                           7.38      \n---------------Inter-token Latency----------------\nMean ITL (ms):                           5.36      \nMedian ITL (ms):                         5.29      \nP99 ITL (ms):                            7.43      \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6e920219e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4730dd07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  183.59    \nTotal input tokens:                      146694    \nTotal generated tokens:                  152861    \nRequest throughput (req/s):              3.92      \nOutput token throughput (tok/s):         832.64    \nPeak output token throughput (tok/s):    1713.00   \nPeak concurrent requests:                25.00     \nTotal Token throughput (tok/s):          1631.69   \n---------------Time to First Token----------------\nMean TTFT (ms):                          29.84     \nMedian TTFT (ms):                        24.09     \nP99 TTFT (ms):                           60.04     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.01      \nMedian TPOT (ms):                        7.77      \nP99 TPOT (ms):                           11.35     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.96      \nMedian ITL (ms):                         7.28      \nP99 ITL (ms):                            27.43     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 133.56136341392994,
+    "num_requests": 1000,
+    "total_num_tokens": 741334,
+    "requests_per_second": 7.487195207051202,
+    "tokens_per_second": 5550.512371624096
+}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fcbc7235c60>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d6b41380-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  185.59    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         209.60    \nPeak output token throughput (tok/s):    517.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          413.50    \n---------------Time to First Token----------------\nMean TTFT (ms):                          39.73     \nMedian TTFT (ms):                        32.58     \nP99 TTFT (ms):                           92.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.35     \nMedian TPOT (ms):                        11.23     \nP99 TPOT (ms):                           13.06     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.32     \nMedian ITL (ms):                         11.05     \nP99 ITL (ms):                            13.41     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5c44aa9c60>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-10085502-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  188.38    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152175    \nRequest throughput (req/s):              3.82      \nOutput token throughput (tok/s):         807.79    \nPeak output token throughput (tok/s):    1510.00   \nPeak concurrent requests:                29.00     \nTotal Token throughput (tok/s):          1581.79   \n---------------Time to First Token----------------\nMean TTFT (ms):                          38.75     \nMedian TTFT (ms):                        31.18     \nP99 TTFT (ms):                           92.33     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          12.77     \nMedian TPOT (ms):                        12.43     \nP99 TPOT (ms):                           17.06     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           12.71     \nMedian ITL (ms):                         11.91     \nP99 ITL (ms):                            36.70     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 230.30906111001968,
+    "num_requests": 1000,
+    "total_num_tokens": 736330,
+    "requests_per_second": 4.341991562035397,
+    "tokens_per_second": 3197.1386468735236
+}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7c0eab59e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7ca49d8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  182.22    \nTotal input tokens:                      38756     \nTotal generated tokens:                  39194     \nRequest throughput (req/s):              0.99      \nOutput token throughput (tok/s):         215.09    \nPeak output token throughput (tok/s):    788.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          427.78    \n---------------Time to First Token----------------\nMean TTFT (ms):                          26.17     \nMedian TTFT (ms):                        21.35     \nP99 TTFT (ms):                           52.39     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          5.08      \nMedian TPOT (ms):                        5.07      \nP99 TPOT (ms):                           7.44      \n---------------Inter-token Latency----------------\nMean ITL (ms):                           5.10      \nMedian ITL (ms):                         5.12      \nP99 ITL (ms):                            7.77      \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8d131e19e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1904b29f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  183.65    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151340    \nRequest throughput (req/s):              3.92      \nOutput token throughput (tok/s):         824.06    \nPeak output token throughput (tok/s):    1801.00   \nPeak concurrent requests:                25.00     \nTotal Token throughput (tok/s):          1616.54   \n---------------Time to First Token----------------\nMean TTFT (ms):                          26.49     \nMedian TTFT (ms):                        22.28     \nP99 TTFT (ms):                           56.70     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.24      \nMedian TPOT (ms):                        8.09      \nP99 TPOT (ms):                           10.51     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.19      \nMedian ITL (ms):                         7.72      \nP99 ITL (ms):                            14.51     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 107.1224673166871,
+    "num_requests": 1000,
+    "total_num_tokens": 738792,
+    "requests_per_second": 9.335109851826799,
+    "tokens_per_second": 6896.704477650824
+}