x

Dockerfile

@@ -1,164 +1,26 @@
-FROM registry.fedoraproject.org/fedora:43
+FROM docker.1ms.run/kyuz0/vllm-therock-gfx1201:latest
 
-# 1. System Base & Build Tools
+# Set environment variables
-# Added 'gperftools-libs' for tcmalloc (fixes double-free)
+ENV USE_DEFAULT_MODEL=true
-RUN dnf -y install --setopt=install_weak_deps=False --nodocs \
+ENV LOCAL_MODEL_DIR=/opt/model
-  python3.13 python3.13-devel git rsync libatomic bash ca-certificates curl \
-  gcc gcc-c++ binutils make ffmpeg-free \
-  cmake ninja-build aria2c tar xz vim nano \
-  libdrm-devel zlib-devel openssl-devel jq \
-  numactl-devel gperftools-libs dialog procps-ng \
-  && dnf clean all && rm -rf /var/cache/dnf/*
 
-# 2. Install "TheRock" ROCm SDK (Tarball Method)
+# Create necessary directories
-WORKDIR /tmp
+RUN mkdir -p /opt/script /opt/model /config
-ARG ROCM_MAJOR_VER=7
-ARG GFX=gfx120X-all
-RUN set -euo pipefail; \
-  BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
-  PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
-  KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
-  | tr '<' '\n' \
-  | grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
-  | sort -V | tail -n1)"; \
-  echo "Downloading Latest Tarball: ${KEY}"; \
-  aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz; \
-  mkdir -p /opt/rocm; \
-  tar xzf therock.tar.gz -C /opt/rocm --strip-components=1; \
-  rm therock.tar.gz
 
-# 3. Configure Global ROCm Environment
+# Copy scripts to /opt/script
-# We add LD_PRELOAD for tcmalloc here to fix the shutdown crash
+COPY scripts/start_vllm.py /opt/script/start-vllm
-RUN export ROCM_PATH=/opt/rocm && \
+COPY benchmarks/run_vllm_bench.py /opt/script/run_vllm_bench.py
-  BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
-  printf '%s\n' \
-  "export ROCM_PATH=/opt/rocm" \
-  "export HIP_PLATFORM=amd" \
-  "export HIP_PATH=/opt/rocm" \
-  "export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \
-  "export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \
-  "export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \
-  "export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \
-  "export ROCBLAS_USE_HIPBLASLT=1" \
-  "export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \
-  "export VLLM_TARGET_DEVICE=rocm" \
-  "export HIP_FORCE_DEV_KERNARG=1" \
-  "export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \
-  "export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \
-  > /etc/profile.d/rocm-sdk.sh && \
-  chmod 0644 /etc/profile.d/rocm-sdk.sh
 
-# 4. Python Venv Setup
+# Note: config.json should be mounted at runtime via docker-compose.yml
-RUN /usr/bin/python3.13 -m venv /opt/venv
-ENV VIRTUAL_ENV=/opt/venv
-ENV PATH=/opt/venv/bin:$PATH
-ENV PIP_NO_CACHE_DIR=1
-RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh
-RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0"
 
-# 5. Install PyTorch (TheRock Nightly)
+# Make scripts executable
-RUN python -m pip install \
+RUN chmod +x /opt/script/start-vllm
-  --index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \
-  --pre torch torchaudio torchvision
 
-# Flash-Attention
+# Replace the container's start-vllm with our improved version
+RUN cp /opt/script/start-vllm /usr/local/bin/start-vllm
+
+# Set working directory
 WORKDIR /opt
-ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
 
-RUN git clone https://github.com/ROCm/flash-attention.git &&\ 
+# Default command
-  cd flash-attention &&\
+CMD ["start-vllm"]
-  git checkout main_perf &&\
-  python setup.py install && \
-  cd /opt && rm -rf /opt/flash-attention
-
-# 6. Clone vLLM
-RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm
-WORKDIR /opt/vllm
-
-# --- PATCHING ---
-# vLLM relies on 'amdsmi' to detect AMD GPUs. If it's missing or fails (common in containers),
-# vLLM falls back to CPU. We patch it to force ROCm detection.
-RUN echo "import sys, re" > patch_vllm.py && \
-  echo "from pathlib import Path" >> patch_vllm.py && \
-  # Patch 1: __init__.py - Force is_rocm=True and bypass amdsmi checks
-  echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \
-  echo "txt = p.read_text()" >> patch_vllm.py && \
-  echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \
-  echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \
-  echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \
-  echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \
-  echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \
-  echo "p.write_text(txt)" >> patch_vllm.py && \
-  # Patch 2: rocm.py - Mock amdsmi and force device name
-  echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \
-  echo "txt = p.read_text()" >> patch_vllm.py && \
-  echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \
-  echo "txt = header + txt" >> patch_vllm.py && \
-  echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \
-  echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \
-  echo "txt += '\n    def get_device_name(self, device_id: int = 0) -> str:\n        return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \
-  echo "p.write_text(txt)" >> patch_vllm.py && \
-  echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \
-  python patch_vllm.py
-
-# 7. Build vLLM (Wheel Method) with CLANG Host Compiler
-RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11
-ENV ROCM_HOME="/opt/rocm"
-ENV HIP_PATH="/opt/rocm"
-ENV VLLM_TARGET_DEVICE="rocm"
-ENV PYTORCH_ROCM_ARCH="gfx1201"
-ENV HIP_ARCHITECTURES="gfx1201"          
-ENV AMDGPU_TARGETS="gfx1201"              
-ENV MAX_JOBS="4"
-
-# --- FIX FOR SEGFAULT ---
-# We force the Host Compiler (CC/CXX) to be the ROCm Clang, not Fedora GCC.
-# This aligns the ABI of the compiled vLLM extensions with PyTorch.
-ENV CC="/opt/rocm/llvm/bin/clang"
-ENV CXX="/opt/rocm/llvm/bin/clang++"
-
-RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
-  echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \
-  export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
-  export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
-  python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \
-  python -m pip install /tmp/dist/*.whl
-
-# --- bitsandbytes (ROCm) ---
-WORKDIR /opt
-RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git
-WORKDIR /opt/bitsandbytes
-
-# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile)
-ENV HIP_PLATFORM="amd"
-ENV CMAKE_PREFIX_PATH="/opt/rocm"
-
-# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++)
-RUN cmake -S . \
-  -DGPU_TARGETS="gfx1201" \
-  -DBNB_ROCM_ARCH="gfx1201" \
-  -DCOMPUTE_BACKEND=hip \
-  -DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
-  -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
-  && \
-  make -j$(nproc) && \
-  python -m pip install --no-cache-dir . --no-build-isolation --no-deps
-
-# 8. Final Cleanup & Runtime
-WORKDIR /opt
-RUN chmod -R a+rwX /opt && \
-  find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \
-  find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \
-  rm -rf /root/.cache/pip || true && \
-  dnf clean all && rm -rf /var/cache/dnf/*
-
-COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh
-COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh
-COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh
-COPY scripts/start_vllm.py /usr/local/bin/start-vllm
-COPY benchmarks/max_context_results.json /opt/max_context_results.json
-COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py
-RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json
-RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
-
-CMD ["/bin/bash"]

README.md

@@ -170,7 +170,7 @@ Once the server is up, hit the OpenAI‑compatible endpoint:
 ```bash
 curl -X POST http://localhost:8000/v1/chat/completions \
   -H "Content-Type: application/json" \
-  -d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"Hello! Test the performance."}]}'
+  -d '{"model":"Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit","messages":[{"role":"user","content":"Hello! Test the performance."}]}'
 ```
 
 You should receive a JSON response with a `choices[0].message.content` reply.
@@ -206,4 +206,3 @@ docker run -p 3000:3000 \
   -v chat-ui-data:/data \
   ghcr.io/huggingface/chat-ui-db
 ```
-

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Llama-3.1-8B-Instruct-FP8-block_tp1_server.log

@@ -1,95 +0,0 @@
-WARNING 12-10 09:53:36 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
-(APIServer pid=4288) INFO 12-10 09:53:36 [api_server.py:1772] vLLM API server version 0.12.0
-(APIServer pid=4288) INFO 12-10 09:53:36 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'host': '127.0.0.1', 'model': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'trust_remote_code': True, 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64}
-(APIServer pid=4288) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
-(APIServer pid=4288) INFO 12-10 09:53:47 [model.py:637] Resolved architecture: LlamaForCausalLM
-(APIServer pid=4288) INFO 12-10 09:53:47 [model.py:1750] Using max model len 65536
-(APIServer pid=4288) INFO 12-10 09:53:47 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
-(APIServer pid=4288) Traceback (most recent call last):
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 409, in hf_raise_for_status
-(APIServer pid=4288)     response.raise_for_status()
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/requests/models.py", line 1026, in raise_for_status
-(APIServer pid=4288)     raise HTTPError(http_error_msg, response=self)
-(APIServer pid=4288) requests.exceptions.HTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block
-(APIServer pid=4288) 
-(APIServer pid=4288) The above exception was the direct cause of the following exception:
-(APIServer pid=4288) 
-(APIServer pid=4288) Traceback (most recent call last):
-(APIServer pid=4288)   File "/venv/main/bin/vllm", line 7, in <module>
-(APIServer pid=4288)     sys.exit(main())
-(APIServer pid=4288)              ^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
-(APIServer pid=4288)     args.dispatch_function(args)
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
-(APIServer pid=4288)     uvloop.run(run_server(args))
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run
-(APIServer pid=4288)     return __asyncio.run(
-(APIServer pid=4288)            ^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/asyncio/runners.py", line 195, in run
-(APIServer pid=4288)     return runner.run(main)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/asyncio/runners.py", line 118, in run
-(APIServer pid=4288)     return self._loop.run_until_complete(task)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper
-(APIServer pid=4288)     return await main
-(APIServer pid=4288)            ^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1819, in run_server
-(APIServer pid=4288)     await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1838, in run_server_worker
-(APIServer pid=4288)     async with build_async_engine_client(
-(APIServer pid=4288)                ^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
-(APIServer pid=4288)     return await anext(self.gen)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 183, in build_async_engine_client
-(APIServer pid=4288)     async with build_async_engine_client_from_engine_args(
-(APIServer pid=4288)                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
-(APIServer pid=4288)     return await anext(self.gen)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 224, in build_async_engine_client_from_engine_args
-(APIServer pid=4288)     async_llm = AsyncLLM.from_vllm_config(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 223, in from_vllm_config
-(APIServer pid=4288)     return cls(
-(APIServer pid=4288)            ^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 114, in __init__
-(APIServer pid=4288)     tokenizer = init_tokenizer_from_config(self.model_config)
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 227, in init_tokenizer_from_config
-(APIServer pid=4288)     return get_tokenizer(
-(APIServer pid=4288)            ^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 191, in get_tokenizer
-(APIServer pid=4288)     tokenizer = TokenizerRegistry.get_tokenizer(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 86, in get_tokenizer
-(APIServer pid=4288)     return item.from_pretrained(*args, **kwargs)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/hf.py", line 84, in from_pretrained
-(APIServer pid=4288)     tokenizer = AutoTokenizer.from_pretrained(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 1156, in from_pretrained
-(APIServer pid=4288)     return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2113, in from_pretrained
-(APIServer pid=4288)     return cls._from_pretrained(
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2395, in _from_pretrained
-(APIServer pid=4288)     tokenizer = cls._patch_mistral_regex(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2438, in _patch_mistral_regex
-(APIServer pid=4288)     if _is_local or is_base_mistral(pretrained_model_name_or_path):
-(APIServer pid=4288)                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2432, in is_base_mistral
-(APIServer pid=4288)     model = model_info(model_id)
-(APIServer pid=4288)             ^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
-(APIServer pid=4288)     return fn(*args, **kwargs)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/hf_api.py", line 2638, in model_info
-(APIServer pid=4288)     hf_raise_for_status(r)
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 482, in hf_raise_for_status
-(APIServer pid=4288)     raise _format(HfHubHTTPError, str(e), response) from e
-(APIServer pid=4288) huggingface_hub.errors.HfHubHTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73dcacabf4c0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f177f1fc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  191.24    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         210.71    \nPeak output token throughput (tok/s):    430.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          411.29    \n---------------Time to First Token----------------\nMean TTFT (ms):                          145.79    \nMedian TTFT (ms):                        110.27    \nP99 TTFT (ms):                           399.40    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.46     \nMedian TPOT (ms):                        22.67     \nP99 TPOT (ms):                           36.03     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.37     \nMedian ITL (ms):                         21.34     \nP99 ITL (ms):                            84.18     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cabb891f240>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31b9a516-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  215.08    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155647    \nRequest throughput (req/s):              3.35      \nOutput token throughput (tok/s):         723.67    \nPeak output token throughput (tok/s):    1208.00   \nPeak concurrent requests:                113.00    \nTotal Token throughput (tok/s):          1405.72   \n---------------Time to First Token----------------\nMean TTFT (ms):                          11329.86  \nMedian TTFT (ms):                        14049.92  \nP99 TTFT (ms):                           20450.77  \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          40.85     \nMedian TPOT (ms):                        39.82     \nP99 TPOT (ms):                           83.14     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           40.32     \nMedian ITL (ms):                         26.70     \nP99 ITL (ms):                            338.67    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 1446.3949257107452,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 0.6913741069083253,
-    "tokens_per_second": 512.5391321707765
-}

benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7efdf61eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-db5fc148-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.17    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40015     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         217.27    \nPeak output token throughput (tok/s):    541.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          425.55    \n---------------Time to First Token----------------\nMean TTFT (ms):                          58.17     \nMedian TTFT (ms):                        40.63     \nP99 TTFT (ms):                           140.17    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          9.73      \nMedian TPOT (ms):                        9.55      \nP99 TPOT (ms):                           15.94     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           9.55      \nMedian ITL (ms):                         9.45      \nP99 ITL (ms):                            14.11     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7aa491426fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c07c837c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  189.59    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153522    \nRequest throughput (req/s):              3.80      \nOutput token throughput (tok/s):         809.76    \nPeak output token throughput (tok/s):    1387.00   \nPeak concurrent requests:                40.00     \nTotal Token throughput (tok/s):          1583.51   \n---------------Time to First Token----------------\nMean TTFT (ms):                          83.62     \nMedian TTFT (ms):                        71.44     \nP99 TTFT (ms):                           196.63    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          22.34     \nMedian TPOT (ms):                        21.97     \nP99 TPOT (ms):                           35.34     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           21.92     \nMedian ITL (ms):                         19.31     \nP99 ITL (ms):                            107.58    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 306.69494965299964,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 3.2605688523121055,
-    "tokens_per_second": 2417.170549559942
-}

benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x78113ee3efc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bec359dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  191.18    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38760     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         202.74    \nPeak output token throughput (tok/s):    418.00    \nPeak concurrent requests:                11.00     \nTotal Token throughput (tok/s):          400.67    \n---------------Time to First Token----------------\nMean TTFT (ms):                          95.72     \nMedian TTFT (ms):                        55.76     \nP99 TTFT (ms):                           257.24    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.29     \nMedian TPOT (ms):                        23.04     \nP99 TPOT (ms):                           28.71     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.29     \nMedian ITL (ms):                         22.01     \nP99 ITL (ms):                            67.09     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7eff14b2efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-acbcf111-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  199.30    \nTotal input tokens:                      145810    \nTotal generated tokens:                  151171    \nRequest throughput (req/s):              3.61      \nOutput token throughput (tok/s):         758.52    \nPeak output token throughput (tok/s):    1326.00   \nPeak concurrent requests:                52.00     \nTotal Token throughput (tok/s):          1490.14   \n---------------Time to First Token----------------\nMean TTFT (ms):                          134.75    \nMedian TTFT (ms):                        130.98    \nP99 TTFT (ms):                           409.51    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          36.74     \nMedian TPOT (ms):                        35.30     \nP99 TPOT (ms):                           77.70     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           35.30     \nMedian ITL (ms):                         26.57     \nP99 ITL (ms):                            192.53    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 402.3294672546908,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 2.4855251265176648,
-    "tokens_per_second": 1830.1667164087519
-}

benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7edefddeafc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b23a48ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.20    \nTotal input tokens:                      38756     \nTotal generated tokens:                  39194     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.78    \nPeak output token throughput (tok/s):    495.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          423.18    \n---------------Time to First Token----------------\nMean TTFT (ms):                          60.23     \nMedian TTFT (ms):                        37.85     \nP99 TTFT (ms):                           146.84    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.21     \nMedian TPOT (ms):                        11.26     \nP99 TPOT (ms):                           18.07     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.00     \nMedian ITL (ms):                         10.95     \nP99 ITL (ms):                            19.93     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7400f615afc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2c7096a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.29    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151404    \nRequest throughput (req/s):              3.78      \nOutput token throughput (tok/s):         795.67    \nPeak output token throughput (tok/s):    1315.00   \nPeak concurrent requests:                36.00     \nTotal Token throughput (tok/s):          1560.51   \n---------------Time to First Token----------------\nMean TTFT (ms):                          65.82     \nMedian TTFT (ms):                        45.53     \nP99 TTFT (ms):                           177.98    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          20.36     \nMedian TPOT (ms):                        20.02     \nP99 TPOT (ms):                           31.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           20.10     \nMedian ITL (ms):                         18.54     \nP99 ITL (ms):                            90.23     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 357.56243500020355,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 2.7967143696161223,
-    "tokens_per_second": 2066.190202557434
-}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x71aab4572fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96888f07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  190.82    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         211.17    \nPeak output token throughput (tok/s):    443.00    \nPeak concurrent requests:                11.00     \nTotal Token throughput (tok/s):          412.18    \n---------------Time to First Token----------------\nMean TTFT (ms):                          64.41     \nMedian TTFT (ms):                        50.82     \nP99 TTFT (ms):                           144.55    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          21.14     \nMedian TPOT (ms):                        20.98     \nP99 TPOT (ms):                           23.59     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           21.12     \nMedian ITL (ms):                         20.65     \nP99 ITL (ms):                            29.23     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x76b9330fafc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bd32e896-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  194.55    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155576    \nRequest throughput (req/s):              3.70      \nOutput token throughput (tok/s):         799.66    \nPeak output token throughput (tok/s):    1300.00   \nPeak concurrent requests:                43.00     \nTotal Token throughput (tok/s):          1553.66   \n---------------Time to First Token----------------\nMean TTFT (ms):                          92.77     \nMedian TTFT (ms):                        68.07     \nP99 TTFT (ms):                           484.98    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          25.78     \nMedian TPOT (ms):                        25.52     \nP99 TPOT (ms):                           34.37     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           25.65     \nMedian ITL (ms):                         23.39     \nP99 ITL (ms):                            86.03     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 889.9994421191514,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 1.1235962099245025,
-    "tokens_per_second": 832.9600726881711
-}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7236816eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4cdb5c8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.35    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39718     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         216.63    \nPeak output token throughput (tok/s):    666.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          425.84    \n---------------Time to First Token----------------\nMean TTFT (ms):                          37.48     \nMedian TTFT (ms):                        30.33     \nP99 TTFT (ms):                           77.17     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          7.36      \nMedian TPOT (ms):                        7.11      \nP99 TPOT (ms):                           11.45     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.26      \nMedian ITL (ms):                         6.53      \nP99 ITL (ms):                            11.05     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73a294576fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-81c0edc1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  186.20    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153033    \nRequest throughput (req/s):              3.87      \nOutput token throughput (tok/s):         821.89    \nPeak output token throughput (tok/s):    1471.00   \nPeak concurrent requests:                30.00     \nTotal Token throughput (tok/s):          1609.74   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.36     \nMedian TTFT (ms):                        42.14     \nP99 TTFT (ms):                           82.58     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          13.89     \nMedian TPOT (ms):                        13.91     \nP99 TPOT (ms):                           17.75     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           13.74     \nMedian ITL (ms):                         13.29     \nP99 ITL (ms):                            38.23     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 215.380199868232,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 4.642952326220295,
-    "tokens_per_second": 3441.9784198061966
-}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x76db7ad72fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-04789567-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  188.87    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.95      \nOutput token throughput (tok/s):         205.96    \nPeak output token throughput (tok/s):    465.00    \nPeak concurrent requests:                10.00     \nTotal Token throughput (tok/s):          406.32    \n---------------Time to First Token----------------\nMean TTFT (ms):                          52.16     \nMedian TTFT (ms):                        44.25     \nP99 TTFT (ms):                           112.79    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          17.68     \nMedian TPOT (ms):                        17.62     \nP99 TPOT (ms):                           19.35     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           17.64     \nMedian ITL (ms):                         17.35     \nP99 ITL (ms):                            23.66     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7b8c3837efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b098e8e8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  193.10    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152091    \nRequest throughput (req/s):              3.73      \nOutput token throughput (tok/s):         787.62    \nPeak output token throughput (tok/s):    1351.00   \nPeak concurrent requests:                39.00     \nTotal Token throughput (tok/s):          1542.71   \n---------------Time to First Token----------------\nMean TTFT (ms):                          57.32     \nMedian TTFT (ms):                        49.20     \nP99 TTFT (ms):                           126.64    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          20.27     \nMedian TPOT (ms):                        20.07     \nP99 TPOT (ms):                           27.95     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           20.15     \nMedian ITL (ms):                         19.30     \nP99 ITL (ms):                            62.76     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 252.42282237578183,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 3.9616069204364575,
-    "tokens_per_second": 2917.0500237249767
-}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7304080eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31ba87dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.12    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38869     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.27    \nPeak output token throughput (tok/s):    609.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          423.91    \n---------------Time to First Token----------------\nMean TTFT (ms):                          40.22     \nMedian TTFT (ms):                        36.83     \nP99 TTFT (ms):                           74.88     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.07      \nMedian TPOT (ms):                        8.23      \nP99 TPOT (ms):                           11.78     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.04      \nMedian ITL (ms):                         8.17      \nP99 ITL (ms):                            12.73     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cdc7797efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b94205ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  187.07    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151811    \nRequest throughput (req/s):              3.85      \nOutput token throughput (tok/s):         811.50    \nPeak output token throughput (tok/s):    1549.00   \nPeak concurrent requests:                30.00     \nTotal Token throughput (tok/s):          1589.49   \n---------------Time to First Token----------------\nMean TTFT (ms):                          41.02     \nMedian TTFT (ms):                        36.91     \nP99 TTFT (ms):                           79.84     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.71     \nMedian TPOT (ms):                        14.74     \nP99 TPOT (ms):                           17.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.60     \nMedian ITL (ms):                         14.44     \nP99 ITL (ms):                            32.49     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 275.10345687624067,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 3.6349961260205634,
-    "tokens_per_second": 2685.5060579349843
-}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f90e18fd9e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0abaed61-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  192.06    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         209.81    \nPeak output token throughput (tok/s):    440.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          409.54    \n---------------Time to First Token----------------\nMean TTFT (ms):                          51.40     \nMedian TTFT (ms):                        47.89     \nP99 TTFT (ms):                           93.19     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.06     \nMedian TPOT (ms):                        23.00     \nP99 TPOT (ms):                           24.12     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.10     \nMedian ITL (ms):                         22.83     \nP99 ITL (ms):                            28.65     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f205d7019e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-14719fc9-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  195.24    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155585    \nRequest throughput (req/s):              3.69      \nOutput token throughput (tok/s):         796.89    \nPeak output token throughput (tok/s):    1321.00   \nPeak concurrent requests:                39.00     \nTotal Token throughput (tok/s):          1548.24   \n---------------Time to First Token----------------\nMean TTFT (ms):                          53.18     \nMedian TTFT (ms):                        49.55     \nP99 TTFT (ms):                           101.04    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          24.41     \nMedian TPOT (ms):                        24.26     \nP99 TPOT (ms):                           28.40     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           24.37     \nMedian ITL (ms):                         23.61     \nP99 ITL (ms):                            56.70     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 228.03156735002995,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 4.385357745074801,
-    "tokens_per_second": 3251.0147985872827
-}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fcbbe1e19e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5741d2ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  182.76    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39611     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         216.74    \nPeak output token throughput (tok/s):    789.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          426.61    \n---------------Time to First Token----------------\nMean TTFT (ms):                          30.25     \nMedian TTFT (ms):                        23.16     \nP99 TTFT (ms):                           57.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          5.38      \nMedian TPOT (ms):                        5.32      \nP99 TPOT (ms):                           7.38      \n---------------Inter-token Latency----------------\nMean ITL (ms):                           5.36      \nMedian ITL (ms):                         5.29      \nP99 ITL (ms):                            7.43      \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6e920219e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4730dd07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  183.59    \nTotal input tokens:                      146694    \nTotal generated tokens:                  152861    \nRequest throughput (req/s):              3.92      \nOutput token throughput (tok/s):         832.64    \nPeak output token throughput (tok/s):    1713.00   \nPeak concurrent requests:                25.00     \nTotal Token throughput (tok/s):          1631.69   \n---------------Time to First Token----------------\nMean TTFT (ms):                          29.84     \nMedian TTFT (ms):                        24.09     \nP99 TTFT (ms):                           60.04     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.01      \nMedian TPOT (ms):                        7.77      \nP99 TPOT (ms):                           11.35     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.96      \nMedian ITL (ms):                         7.28      \nP99 ITL (ms):                            27.43     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 133.56136341392994,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 7.487195207051202,
-    "tokens_per_second": 5550.512371624096
-}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fcbc7235c60>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d6b41380-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  185.59    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         209.60    \nPeak output token throughput (tok/s):    517.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          413.50    \n---------------Time to First Token----------------\nMean TTFT (ms):                          39.73     \nMedian TTFT (ms):                        32.58     \nP99 TTFT (ms):                           92.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.35     \nMedian TPOT (ms):                        11.23     \nP99 TPOT (ms):                           13.06     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.32     \nMedian ITL (ms):                         11.05     \nP99 ITL (ms):                            13.41     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5c44aa9c60>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-10085502-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  188.38    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152175    \nRequest throughput (req/s):              3.82      \nOutput token throughput (tok/s):         807.79    \nPeak output token throughput (tok/s):    1510.00   \nPeak concurrent requests:                29.00     \nTotal Token throughput (tok/s):          1581.79   \n---------------Time to First Token----------------\nMean TTFT (ms):                          38.75     \nMedian TTFT (ms):                        31.18     \nP99 TTFT (ms):                           92.33     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          12.77     \nMedian TPOT (ms):                        12.43     \nP99 TPOT (ms):                           17.06     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           12.71     \nMedian ITL (ms):                         11.91     \nP99 ITL (ms):                            36.70     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 230.30906111001968,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 4.341991562035397,
-    "tokens_per_second": 3197.1386468735236
-}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7c0eab59e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7ca49d8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  182.22    \nTotal input tokens:                      38756     \nTotal generated tokens:                  39194     \nRequest throughput (req/s):              0.99      \nOutput token throughput (tok/s):         215.09    \nPeak output token throughput (tok/s):    788.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          427.78    \n---------------Time to First Token----------------\nMean TTFT (ms):                          26.17     \nMedian TTFT (ms):                        21.35     \nP99 TTFT (ms):                           52.39     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          5.08      \nMedian TPOT (ms):                        5.07      \nP99 TPOT (ms):                           7.44      \n---------------Inter-token Latency----------------\nMean ITL (ms):                           5.10      \nMedian ITL (ms):                         5.12      \nP99 ITL (ms):                            7.77      \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8d131e19e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1904b29f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  183.65    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151340    \nRequest throughput (req/s):              3.92      \nOutput token throughput (tok/s):         824.06    \nPeak output token throughput (tok/s):    1801.00   \nPeak concurrent requests:                25.00     \nTotal Token throughput (tok/s):          1616.54   \n---------------Time to First Token----------------\nMean TTFT (ms):                          26.49     \nMedian TTFT (ms):                        22.28     \nP99 TTFT (ms):                           56.70     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.24      \nMedian TPOT (ms):                        8.09      \nP99 TPOT (ms):                           10.51     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.19      \nMedian ITL (ms):                         7.72      \nP99 ITL (ms):                            14.51     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 107.1224673166871,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 9.335109851826799,
-    "tokens_per_second": 6896.704477650824
-}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f88cf679990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-66e0ceca-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  187.88    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40286     \nRequest throughput (req/s):              0.96      \nOutput token throughput (tok/s):         214.42    \nPeak output token throughput (tok/s):    460.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          418.59    \n---------------Time to First Token----------------\nMean TTFT (ms):                          77.11     \nMedian TTFT (ms):                        52.45     \nP99 TTFT (ms):                           199.22    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          15.76     \nMedian TPOT (ms):                        15.43     \nP99 TPOT (ms):                           21.30     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           15.60     \nMedian ITL (ms):                         14.96     \nP99 ITL (ms):                            17.95     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f719e189990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-702c15c4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.21    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155632    \nRequest throughput (req/s):              3.79      \nOutput token throughput (tok/s):         818.19    \nPeak output token throughput (tok/s):    1520.00   \nPeak concurrent requests:                37.00     \nTotal Token throughput (tok/s):          1589.40   \n---------------Time to First Token----------------\nMean TTFT (ms):                          76.15     \nMedian TTFT (ms):                        45.58     \nP99 TTFT (ms):                           247.34    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          19.54     \nMedian TPOT (ms):                        18.83     \nP99 TPOT (ms):                           35.16     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           19.12     \nMedian ITL (ms):                         16.70     \nP99 ITL (ms):                            114.66    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 248.4086626805365,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 4.0256245060424485,
-    "tokens_per_second": 2984.3323175624723
-}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc3c8c05990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b80b1bb5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.14    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39211     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.94    \nPeak output token throughput (tok/s):    560.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          421.26    \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.12     \nMedian TTFT (ms):                        41.56     \nP99 TTFT (ms):                           82.51     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.89      \nMedian TPOT (ms):                        8.73      \nP99 TPOT (ms):                           13.87     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.70      \nMedian ITL (ms):                         8.44      \nP99 ITL (ms):                            11.78     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ff610411990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fb44348e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  186.80    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153333    \nRequest throughput (req/s):              3.85      \nOutput token throughput (tok/s):         820.85    \nPeak output token throughput (tok/s):    1449.00   \nPeak concurrent requests:                31.00     \nTotal Token throughput (tok/s):          1606.16   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.68     \nMedian TTFT (ms):                        36.77     \nP99 TTFT (ms):                           89.21     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.46     \nMedian TPOT (ms):                        14.20     \nP99 TPOT (ms):                           19.40     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.28     \nMedian ITL (ms):                         13.63     \nP99 ITL (ms):                            44.44     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 199.3360565919429,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 5.016653871341908,
-    "tokens_per_second": 3719.016081057382
-}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fe9a3b39990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4225f4cd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  187.19    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38830     \nRequest throughput (req/s):              0.96      \nOutput token throughput (tok/s):         207.43    \nPeak output token throughput (tok/s):    490.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          409.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          45.84     \nMedian TTFT (ms):                        37.87     \nP99 TTFT (ms):                           101.24    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.35     \nMedian TPOT (ms):                        14.27     \nP99 TPOT (ms):                           16.03     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.28     \nMedian ITL (ms):                         14.04     \nP99 ITL (ms):                            16.81     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f45831f1990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ca67e7d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.55    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152171    \nRequest throughput (req/s):              3.78      \nOutput token throughput (tok/s):         798.60    \nPeak output token throughput (tok/s):    1492.00   \nPeak concurrent requests:                35.00     \nTotal Token throughput (tok/s):          1563.82   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.19     \nMedian TTFT (ms):                        37.54     \nP99 TTFT (ms):                           101.54    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          16.05     \nMedian TPOT (ms):                        15.69     \nP99 TPOT (ms):                           20.99     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           15.95     \nMedian ITL (ms):                         15.02     \nP99 ITL (ms):                            41.81     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 195.21851312741637,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 5.122464995660089,
-    "tokens_per_second": 3771.8246502543934
-}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc0cf5b5990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-85d6e157-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.05    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38779     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         211.85    \nPeak output token throughput (tok/s):    676.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          423.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          36.19     \nMedian TTFT (ms):                        30.20     \nP99 TTFT (ms):                           83.37     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          7.40      \nMedian TPOT (ms):                        7.32      \nP99 TPOT (ms):                           10.60     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.34      \nMedian ITL (ms):                         7.07      \nP99 ITL (ms):                            10.69     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6542301990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-584a872e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  185.93    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151690    \nRequest throughput (req/s):              3.87      \nOutput token throughput (tok/s):         815.86    \nPeak output token throughput (tok/s):    1630.00   \nPeak concurrent requests:                28.00     \nTotal Token throughput (tok/s):          1598.63   \n---------------Time to First Token----------------\nMean TTFT (ms):                          38.36     \nMedian TTFT (ms):                        31.91     \nP99 TTFT (ms):                           78.67     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          12.67     \nMedian TPOT (ms):                        12.82     \nP99 TPOT (ms):                           16.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           12.51     \nMedian ITL (ms):                         12.62     \nP99 ITL (ms):                            31.79     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 156.32366928458214,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 6.396983928131399,
-    "tokens_per_second": 4726.040550232053
-}

benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7d8533bc6fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-39a9e102-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  197.48    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.91      \nOutput token throughput (tok/s):         204.06    \nPeak output token throughput (tok/s):    372.00    \nPeak concurrent requests:                15.00     \nTotal Token throughput (tok/s):          398.30    \n---------------Time to First Token----------------\nMean TTFT (ms):                          78.01     \nMedian TTFT (ms):                        70.55     \nP99 TTFT (ms):                           149.81    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          34.15     \nMedian TPOT (ms):                        34.06     \nP99 TPOT (ms):                           36.67     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           34.16     \nMedian ITL (ms):                         33.55     \nP99 ITL (ms):                            58.95     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ad99aa1efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0e099965-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  205.53    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155647    \nRequest throughput (req/s):              3.50      \nOutput token throughput (tok/s):         757.31    \nPeak output token throughput (tok/s):    1260.00   \nPeak concurrent requests:                59.00     \nTotal Token throughput (tok/s):          1471.06   \n---------------Time to First Token----------------\nMean TTFT (ms):                          85.03     \nMedian TTFT (ms):                        76.62     \nP99 TTFT (ms):                           176.99    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          41.66     \nMedian TPOT (ms):                        41.32     \nP99 TPOT (ms):                           52.93     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           41.47     \nMedian ITL (ms):                         39.19     \nP99 ITL (ms):                            111.74    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 405.9800275520247,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 2.463175358723414,
-    "tokens_per_second": 1826.0356413838635
-}

benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x74f226dff240>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-341eb0ef-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.76    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40087     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         216.97    \nPeak output token throughput (tok/s):    453.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          424.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          40.37     \nMedian TTFT (ms):                        37.70     \nP99 TTFT (ms):                           70.13     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.50     \nMedian TPOT (ms):                        11.56     \nP99 TPOT (ms):                           16.13     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.24     \nMedian ITL (ms):                         11.23     \nP99 ITL (ms):                            19.54     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73d7f5083240>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-34321bd4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  192.26    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153536    \nRequest throughput (req/s):              3.74      \nOutput token throughput (tok/s):         798.60    \nPeak output token throughput (tok/s):    1194.00   \nPeak concurrent requests:                46.00     \nTotal Token throughput (tok/s):          1561.61   \n---------------Time to First Token----------------\nMean TTFT (ms):                          54.49     \nMedian TTFT (ms):                        52.17     \nP99 TTFT (ms):                           97.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          25.72     \nMedian TPOT (ms):                        25.55     \nP99 TPOT (ms):                           31.80     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           25.55     \nMedian ITL (ms):                         24.86     \nP99 ITL (ms):                            53.01     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 307.8667154959985,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 3.248158861177695,
-    "tokens_per_second": 2407.9706011923054
-}

benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7bbdd0516fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f41559c3-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  195.60    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.92      \nOutput token throughput (tok/s):         198.88    \nPeak output token throughput (tok/s):    395.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          392.34    \n---------------Time to First Token----------------\nMean TTFT (ms):                          63.33     \nMedian TTFT (ms):                        59.42     \nP99 TTFT (ms):                           123.97    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          30.79     \nMedian TPOT (ms):                        30.76     \nP99 TPOT (ms):                           32.51     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           30.78     \nMedian ITL (ms):                         30.39     \nP99 ITL (ms):                            42.55     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7b7e08676fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f456f8a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     719       \nFailed requests:                         1         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  204.29    \nTotal input tokens:                      145785    \nTotal generated tokens:                  151865    \nRequest throughput (req/s):              3.52      \nOutput token throughput (tok/s):         743.36    \nPeak output token throughput (tok/s):    1197.00   \nPeak concurrent requests:                52.00     \nTotal Token throughput (tok/s):          1456.96   \n---------------Time to First Token----------------\nMean TTFT (ms):                          68.23     \nMedian TTFT (ms):                        63.70     \nP99 TTFT (ms):                           136.63    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          34.84     \nMedian TPOT (ms):                        34.66     \nP99 TPOT (ms):                           41.64     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           34.70     \nMedian ITL (ms):                         33.44     \nP99 ITL (ms):                            79.44     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 342.5061867629993,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 2.9196552898822827,
-    "tokens_per_second": 2149.8297795990215
-}

benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fe04afbf240>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-68841ca7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  185.44    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38777     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         209.11    \nPeak output token throughput (tok/s):    437.00    \nPeak concurrent requests:                10.00     \nTotal Token throughput (tok/s):          418.10    \n---------------Time to First Token----------------\nMean TTFT (ms):                          42.77     \nMedian TTFT (ms):                        40.02     \nP99 TTFT (ms):                           73.37     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.89     \nMedian TPOT (ms):                        14.86     \nP99 TPOT (ms):                           21.46     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.57     \nMedian ITL (ms):                         14.54     \nP99 ITL (ms):                            24.86     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x75d3437befc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96562708-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  194.32    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151349    \nRequest throughput (req/s):              3.71      \nOutput token throughput (tok/s):         778.86    \nPeak output token throughput (tok/s):    1218.00   \nPeak concurrent requests:                42.00     \nTotal Token throughput (tok/s):          1527.82   \n---------------Time to First Token----------------\nMean TTFT (ms):                          51.01     \nMedian TTFT (ms):                        49.88     \nP99 TTFT (ms):                           83.96     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          26.15     \nMedian TPOT (ms):                        26.20     \nP99 TPOT (ms):                           28.93     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           26.04     \nMedian ITL (ms):                         25.85     \nP99 ITL (ms):                            42.96     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 256.2449242460134,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 3.9025163247328507,
-    "tokens_per_second": 2883.1478405820326
-}

config.json

@@ -0,0 +1,139 @@
+{
+  "default_model": "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
+  "models": {
+    "Meta-Llama-3.1-8B-Instruct": {
+      "ctx": "65536",
+      "trust_remote": false,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Meta-Llama-3.1-8B-Instruct",
+      "hf_model_id": "meta-llama/Meta-Llama-3.1-8B-Instruct"
+    },
+    "gpt-oss-20b": {
+      "ctx": "32768",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "8192",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "gpt-oss-20b",
+      "hf_model_id": "openai/gpt-oss-20b"
+    },
+    "Qwen3-14B-FP8-dynamic": {
+      "ctx": "32768",
+      "trust_remote": true,
+      "valid_tp": [1],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Qwen3-14B-FP8-dynamic",
+      "hf_model_id": "RedHatAI/Qwen3-14B-FP8-dynamic"
+    },
+    "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
+      "ctx": "131072",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "32",
+      "max_tokens": "65536",
+      "gpu_util": "0.95",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
+      "hf_model_id": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit"
+    },
+    "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
+      "ctx": "24576",
+      "trust_remote": true,
+      "valid_tp": [2],
+      "max_num_seqs": "32",
+      "max_tokens": "16384",
+      "gpu_util": "0.98",
+      "enforce_eager": false,
+      "env": {"VLLM_USE_TRITON_AWQ": "1"},
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
+      "hf_model_id": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit"
+    },
+    "gemma-3-27b-it-FP8-dynamic": {
+      "ctx": "29000",
+      "trust_remote": true,
+      "valid_tp": [2],
+      "max_num_seqs": "32",
+      "max_tokens": "29000",
+      "gpu_util": "0.94",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "gemma-3-27b-it-FP8-dynamic",
+      "hf_model_id": "RedHatAI/gemma-3-27b-it-FP8-dynamic"
+    },
+    "gemma-3-12b-it-FP8-dynamic": {
+      "ctx": "9900",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "9900",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "gemma-3-12b-it-FP8-dynamic",
+      "hf_model_id": "RedHatAI/gemma-3-12b-it-FP8-dynamic"
+    },
+    "GLM-4.7-Flash-AWQ": {
+      "ctx": "32768",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "GLM-4.7-Flash-AWQ",
+      "hf_model_id": "THUDM/GLM-4.7-Flash-AWQ"
+    },
+    "Qwen3.5-27B-FP8": {
+      "ctx": "32768",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Qwen3.5-27B-FP8",
+      "hf_model_id": "RedHatAI/Qwen3.5-27B-FP8-dynamic"
+    },
+    "Qwen3.5-35B-A3B-GPTQ-Int4": {
+      "ctx": "32768",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4",
+      "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
+    },
+    "Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit": {
+      "ctx": "24576",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "tool_call_parser": "qwen3_xml",
+      "enable_auto_tool_choice": true,
+      "served_model_name": "Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit",
+      "hf_model_id": "Qwen/Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit"
+    }
+  }
+}

docker-compose.yml

@@ -0,0 +1,28 @@
+services:
+  vllm:
+    build: .
+    container_name: vllm-r9700
+    network_mode: "host"
+    shm_size: "16gb"
+    environment:
+      - USE_DEFAULT_MODEL=true
+      - LOCAL_MODEL_DIR=/opt/model
+      - HIP_VISIBLE_DEVICES=0,1
+      - NCCL_DEBUG=INFO
+      - NCCL_SOCKET_IFNAME=^lo,docker0
+      - NCCL_P2P_DISABLE=0
+      - NCCL_SHM_DISABLE=0
+      - NCCL_IB_DISABLE=1
+      - NCCL_P2P_LEVEL=SYS
+    volumes:
+      - ./config.json:/config/config.json:ro
+      - /opt/model:/opt/model
+    devices:
+      - /dev/dri:/dev/dri
+      - /dev/kfd:/dev/kfd
+    group_add:
+      - video
+      - render
+    security_opt:
+      - seccomp=unconfined
+    restart: unless-stopped

scripts/add_qwen3_5_moe_support.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Add qwen3_5_moe architecture support to Transformers
+"""
+import os
+import sys
+
+def add_qwen3_5_moe_support():
+    """Add qwen3_5_moe to Transformers configuration"""
+    
+    # Find transformers installation path
+    try:
+        import transformers
+        transformers_path = transformers.__path__[0]
+    except ImportError:
+        print("Transformers not installed")
+        return False
+    
+    print(f"Transformers path: {transformers_path}")
+    
+    # Modify configuration_auto.py
+    config_auto_file = os.path.join(transformers_path, "models", "auto", "configuration_auto.py")
+    
+    if not os.path.exists(config_auto_file):
+        print(f"Config file not found: {config_auto_file}")
+        return False
+    
+    with open(config_auto_file, "r") as f:
+        content = f.read()
+    
+    # Check if already exists
+    if "qwen3_5_moe" in content:
+        print("qwen3_5_moe already exists in configuration")
+        return True
+    
+    # Add to MODEL_NAMES_MAPPING
+    # Find the line with "qwen2_moe": "Qwen2Moe" and add qwen3_5_moe after it
+    if '"qwen2_moe": "Qwen2Moe"' in content:
+        content = content.replace(
+            '"qwen2_moe": "Qwen2Moe"',
+            '"qwen2_moe": "Qwen2Moe",\n        "qwen3_5_moe": "Qwen3_5_MoE"'
+        )
+        print("Added qwen3_5_moe to MODEL_NAMES_MAPPING")
+    
+    # Add to MODEL_MAPPING (mapping to Qwen2MoeConfig as base)
+    if '"qwen2_moe": Qwen2MoeConfig' in content:
+        content = content.replace(
+            '"qwen2_moe": Qwen2MoeConfig',
+            '"qwen2_moe": Qwen2MoeConfig,\n        "qwen3_5_moe": Qwen2MoeConfig'
+        )
+        print("Added qwen3_5_moe to MODEL_MAPPING")
+    
+    with open(config_auto_file, "w") as f:
+        f.write(content)
+    
+    print("Successfully added qwen3_5_moe support")
+    return True
+
+if __name__ == "__main__":
+    success = add_qwen3_5_moe_support()
+    sys.exit(0 if success else 1)

scripts/run_vllm_bench.py

@@ -1,3 +1,4 @@
+cat run_vllm_bench.py 
 #!/usr/bin/env python3
 import subprocess, time, json, sys, os, requests, re, argparse
 from pathlib import Path
@@ -6,11 +7,15 @@ from pathlib import Path
 # ⚙️ GLOBAL SETTINGS
 # =========================
 
-# HARDWARE: NVIDIA GPUs (Auto-detected)
+# HARDWARE: 2x AMD Radeon AI PRO R9700 (32GB, RDNA 4)
-GPU_UTIL = "0.95" 
+GPU_UTIL = "0.98" 
 PORT     = 8000
 HOST     = "127.0.0.1"
 
+# BENCHMARK TOGGLES
+# AITER is disabled/removed.
+ 
+
 # 1. THROUGHPUT CONFIG
 OFF_NUM_PROMPTS      = 1000 
 OFF_FORCED_OUTPUT    = "512"
@@ -25,7 +30,7 @@ QPS_SWEEP       = [1.0, 4.0]
 FALLBACK_INPUT_LEN  = 1024
 FALLBACK_OUTPUT_LEN = 512
 
-RESULTS_DIR = Path("benchmark_results_nvidia")
+RESULTS_DIR = Path("benchmark_results")
 RESULTS_DIR.mkdir(exist_ok=True)
 
 # =========================
@@ -34,6 +39,7 @@ RESULTS_DIR.mkdir(exist_ok=True)
 
 MODEL_TABLE = {
     # 1. Llama 3.1 8B Instruct
+    # MAD uses 131k tokens. We scale to 32k for 32GB VRAM safety.
     "meta-llama/Meta-Llama-3.1-8B-Instruct": {
         "ctx": "65536",  
         "trust_remote": False,
@@ -43,22 +49,23 @@ MODEL_TABLE = {
     },
     
     # 2. GPT-OSS 20B (MXFP4)
+    # MAD Row 0 uses 8192. We match this exactly.
     "openai/gpt-oss-20b": {
         "ctx": "32768", 
         "trust_remote": True,
         "valid_tp": [1, 2],
         "max_num_seqs": "64",
-        "max_tokens": "8192",
+        "max_tokens": "8192"
     },
 
     # 3. Qwen 14B FP8
+    # MAD uses 40k. We use 32k.
     "RedHatAI/Qwen3-14B-FP8-dynamic": {
         "ctx": "32768", 
         "trust_remote": True,
         "valid_tp": [1],
         "max_num_seqs": "64",
-        "max_tokens": "32768",
+        "max_tokens": "32768"
-        "gpu_util": "0.90"
     },
 
     # 4. Qwen 30B 4-bit
@@ -67,35 +74,39 @@ MODEL_TABLE = {
         "trust_remote": True,
         "valid_tp": [1, 2],
         "max_num_seqs": "64",
-        "max_tokens": "32768",
+        "max_tokens": "32768"
-        "gpu_util": "0.90"
     },
 
-    # 5. Qwen 80B AWQ
+    # 5. Qwen 80B AWQ (The Big One) [NEW]
+    # Size: ~48GB. Fits on 2x32GB (64GB). Leftover for Cache: ~16GB.
+    # Config: 20k ctx fits in that cache. Eager mode required for stability.
     "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
         "ctx": "20480", 
         "trust_remote": True,
-        "valid_tp": [2], # Requires 2 GPUs
+        "valid_tp": [2], # Too big for single GPU
-        "max_num_seqs": "32", 
+        "max_num_seqs": "32", # Lower concurrency for safety
-        "max_tokens": "16384",
+        "max_tokens": "16384", # Lower batch size because Eager mode is CPU intensive
+        "enforce_eager": False, 
+        "env": {"VLLM_USE_TRITON_AWQ": "1"} # Fixes "Unsupported Hardware" error
     },
 
-    # 6. Llama 3.1 8B FP8
+    # 76 Gemma 3 27B FP8
-    "RedHatAI/Llama-3.1-8B-Instruct-FP8-block": {
+    "RedHatAI/gemma-3-27b-it-FP8-dynamic": {
-        "ctx": "65536",
+        "ctx": "29000",
         "trust_remote": True,
-        "valid_tp": [1, 2],
+        "valid_tp": [2],
-        "max_num_seqs": "64",
+        "max_num_seqs": "32",
-        "max_tokens": "32768",
+        "max_tokens": "29000",
+        "gpu_util": "0.94",
     },
 
     # 7. Gemma 3 12B FP8
     "RedHatAI/gemma-3-12b-it-FP8-dynamic": {
-        "ctx": "32768",
+        "ctx": "9900",
         "trust_remote": True,
         "valid_tp": [1, 2],
         "max_num_seqs": "64",
-        "max_tokens": "32768",
+        "max_tokens": "9900",
     },
 }
 
@@ -105,6 +116,7 @@ MODELS_TO_RUN = [
     "RedHatAI/Qwen3-14B-FP8-dynamic",
     "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
     "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
+    "RedHatAI/gemma-3-27b-it-FP8-dynamic",
     "RedHatAI/gemma-3-12b-it-FP8-dynamic",
 ]
 
@@ -116,33 +128,28 @@ def log(msg): print(f"\n[BENCH] {msg}")
 
 def get_gpu_count():
     try:
-        # Using nvidia-smi -L to list GPUs
+        # Using rocm-smi --showid to list GPUs. 
-        res = subprocess.run(["nvidia-smi", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        # Output format: "GPU[0] : Device Name: ..."
+        res = subprocess.run(["rocm-smi", "--showid"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
         if res.returncode == 0:
-            count = len([line for line in res.stdout.strip().split('\n') if line.strip()])
+            # Filter specifically for the target GPU as requested
-            return count
+            target_gpu = "AMD Radeon AI PRO R9700"
+            count = 0
+            for line in res.stdout.strip().split('\n'):
+                if "Device Name" in line and target_gpu in line:
+                    count += 1
+            
+            return count if count > 0 else 1
         else:
-            log("nvidia-smi failed, defaulting to 1 GPU")
+            log("rocm-smi failed, defaulting to 2 GPUs (Hardcoded Fallback)")
-            return 1
+            return 2
     except Exception as e:
-        log(f"Error detecting GPUs: {e}, defaulting to 1 GPU")
+        log(f"Error detecting GPUs: {e}, defaulting to 2 GPUs")
-        return 1
+        return 2
 
-def force_gpu_cleanup():
+def kill_vllm():
-    """Simple cleanup: just kill vllm processes (excluding self)."""
+    subprocess.run("pgrep -f 'vllm serve' | xargs -r kill -9", 
-    try:
+                   shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        my_pid = os.getpid()
-        # Kill everything matching vllm EXCEPT this process
-        subprocess.run(f"pgrep -f 'vllm' | grep -v {my_pid} | xargs -r kill -9", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        
-        # Original cleanups for other helpers
-        subprocess.run("pkill -9 -f 'multiprocessing'", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        subprocess.run("pkill -9 -f 'resource_tracker'", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        
-        # Try finding fuser to kill processes attached to device files
-        if subprocess.run("which fuser", shell=True, stdout=subprocess.DEVNULL).returncode == 0:
-             subprocess.run("fuser -k -9 /dev/nvidia*", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except: pass
     time.sleep(5)
 
 def nuke_vllm_cache():
@@ -185,54 +192,19 @@ def wait_for_server(url, process, timeout=600):
         time.sleep(2)
     return False
 
-# =========================
-# HARDWARE DETECTION (24GB vs 32GB)
-# =========================
-def is_24gb_card():
-    try:
-        res = subprocess.run(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], 
-                             capture_output=True, text=True)
-        # Check first GPU memory
-        mem = int(res.stdout.strip().split('\n')[0])
-        return mem < 28000 # 4090 is ~24576
-    except:
-        return False
-
-IS_24GB = is_24gb_card()
-if IS_24GB: log("Detected 24GB GPU class (e.g. RTX 4090). Applying memory overrides.")
-else: log("Detected 32GB+ GPU class. Using standard config.")
-
 def get_model_args(model, tp_size):
     config = MODEL_TABLE.get(model, {"ctx": "8192", "max_num_seqs": "32"})
     
-    current_ctx = config["ctx"]
+    # Allow per-model GPU utilization override
-    current_seqs = config["max_num_seqs"]
+    util = config.get("gpu_util", GPU_UTIL)
-    util = None
-
-    if IS_24GB:
-        if model == "meta-llama/Meta-Llama-3.1-8B-Instruct":
-            current_ctx = "31800"
-            log(f"Override: Llama 8B ctx reduced to {current_ctx} for 24GB VRAM")
-        elif model == "openai/gpt-oss-20b":
-            current_ctx = "16384"
-            current_seqs = "32"
-            util = "0.90"
-            log(f"Override: GPT-20B ctx reduced to {current_ctx}, seqs to {current_seqs}, util to {util} for 24GB VRAM")
-        elif model == "RedHatAI/Qwen3-14B-FP8-dynamic":
-            current_ctx = "4096"
-            current_seqs = "32"
-            util = "0.86"
-            log(f"Override: Qwen 14B ctx reduced to {current_ctx}, seqs to {current_seqs}, util to {util} for 24GB VRAM")
-    
-    if util is None: util = config.get("gpu_util", GPU_UTIL)
 
     cmd = [
         "--model", model,
         "--gpu-memory-utilization", util,
-        "--max-model-len", current_ctx, 
+        "--max-model-len", config["ctx"], 
         "--dtype", "auto",
         "--tensor-parallel-size", str(tp_size),
-        "--max-num-seqs", current_seqs
+        "--max-num-seqs", config["max_num_seqs"]
     ]
     
     if config.get("trust_remote"): cmd.append("--trust-remote-code")
@@ -253,10 +225,11 @@ def run_throughput(model, tp_size):
     dataset_path = get_dataset()
     dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"]
     
+    # Retrieve Model-Specific Batch Tokens
     batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS)
 
     log(f"START Throughput {model} (TP={tp_size}) [Batch: {batch_tokens}]...")
-    force_gpu_cleanup()
+    kill_vllm()
     nuke_vllm_cache()
 
     cmd = ["vllm", "bench", "throughput"] + get_model_args(model, tp_size)
@@ -269,14 +242,13 @@ def run_throughput(model, tp_size):
     ])
     cmd.extend(dataset_args)
 
+    # ENV Setup: Global + Model Specific
     env = os.environ.copy()
-    env["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
+    
+    # Inject model specific env vars (e.g. for AWQ)
     model_env = MODEL_TABLE[model].get("env", {})
     env.update(model_env)
 
-    ids_cmd = " ".join(cmd)
-    log(f"CMD: {ids_cmd}")
-
     try: 
         subprocess.run(cmd, check=True, env=env)
     except: 
@@ -291,27 +263,20 @@ def run_latency(model, tp_size):
 
     dataset_path = get_dataset()
     log(f"START Server {model} (TP={tp_size})...")
-    force_gpu_cleanup()
+    kill_vllm()
     nuke_vllm_cache()
 
-
     srv_log = open(RESULTS_DIR / f"{model_safe}_tp{tp_size}_server.log", "w")
+    srv_args = [x for x in get_model_args(model, tp_size) if x != "--model" and x != model]
     
-    # Use get_model_args directly. It includes ["--model", model, ...]
+    # ENV Setup: Global + Model Specific
-    srv_args = get_model_args(model, tp_size) 
-    
     env = os.environ.copy()
-    env["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
+
     model_env = MODEL_TABLE[model].get("env", {})
     env.update(model_env)
 
-    # Command: vllm serve --model <model> ... (no positional model arg)
+    proc = subprocess.Popen(["vllm", "serve", model] + srv_args + ["--host", HOST, "--port", str(PORT)], 
-    cmd = ["vllm", "serve"] + srv_args + ["--host", HOST, "--port", str(PORT)]
+                            stdout=srv_log, stderr=srv_log, env=env)
-    
-    ids_cmd = " ".join(cmd)
-    log(f"CMD: {ids_cmd}")
-
-    proc = subprocess.Popen(cmd, stdout=srv_log, stderr=srv_log, env=env)
 
     try:
         if not wait_for_server(f"http://{HOST}:{PORT}", proc): return
@@ -340,7 +305,7 @@ def run_latency(model, tp_size):
     except Exception as e: log(f"CRASH: {e}")
     finally:
         proc.terminate()
-        force_gpu_cleanup()
+        kill_vllm()
 
 def print_summary(tps):
     print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}")
@@ -377,16 +342,15 @@ if __name__ == "__main__":
     args = parser.parse_args()
     
     gpu_count = get_gpu_count()
-    log(f"Detected {gpu_count} GPU(s)")
+    log(f"Detected {gpu_count} AMD GPU(s)")
     
     valid_tp_args = [t for t in args.tp if t <= gpu_count]
     if not valid_tp_args:
         log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.")
         sys.exit(0)
 
-    force_gpu_cleanup()
+    kill_vllm()
     for tp in valid_tp_args:
         for m in MODELS_TO_RUN:
             run_throughput(m, tp)
             run_latency(m, tp)
-    print_summary(valid_tp_args)

scripts/start_vllm.py

@@ -3,356 +3,250 @@ import sys
 import os
 import json
 import shutil
-import tempfile
 import subprocess
 from pathlib import Path
 
-# Add benchmarks dir to path to import config
+# Configuration
-# Add benchmarks dir to path to import config
+SCRIPT_DIR = Path("/opt/script")
-SCRIPT_DIR = Path(__file__).parent.resolve()
+CONFIG_PATH = Path("/config/config.json")
-BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
+LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", "/opt/model")
-OPT_DIR = Path("/opt")
-
-# Optional environment variable pointing to a local models directory.
-# If set, the script will prefer a subfolder under this path matching
-# the model repo ID (e.g. LOCAL_MODEL_DIR/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit)
-# when constructing the `vllm serve` command.
-LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR")
-
-
-# Check /opt first (Container), then local fallback
-if (OPT_DIR / "run_vllm_bench.py").exists():
-    sys.path.append(str(OPT_DIR))
-else:
-    sys.path.append(str(BENCH_DIR))
-
-try:
-    from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
-except ImportError:
-    print("Error: Could not import run_vllm_bench.py config.")
-    sys.exit(1)
-
-if (OPT_DIR / "max_context_results.json").exists():
-    RESULTS_FILE = OPT_DIR / "max_context_results.json"
-else:
-    RESULTS_FILE = BENCH_DIR / "max_context_results.json"
 HOST = os.getenv("HOST", "0.0.0.0")
 PORT = os.getenv("PORT", "8000")
 
-def check_dependencies():
+def log(msg):
-    if not shutil.which("dialog"):
+    """Print log message with timestamp"""
-        print("Error: 'dialog' is required. Please install it (apt-get install dialog).")
+    print(f"[START-VLLM] {msg}", flush=True)
+
+def load_config():
+    """Load configuration from config.json"""
+    log(f"Loading config from {CONFIG_PATH}")
+    
+    if not CONFIG_PATH.exists():
+        log(f"ERROR: Config file not found at {CONFIG_PATH}")
+        sys.exit(1)
+    
+    try:
+        with open(CONFIG_PATH, "r") as f:
+            config_data = json.load(f)
+        
+        model_table = config_data["models"]
+        default_model = config_data["default_model"]
+        models_to_run = list(model_table.keys())
+        
+        log(f"Loaded {len(models_to_run)} models from config")
+        log(f"Default model: {default_model}")
+        
+        return model_table, default_model, models_to_run
+    except Exception as e:
+        log(f"ERROR: Failed to load config: {e}")
+        import traceback
+        traceback.print_exc()
         sys.exit(1)
 
 def detect_gpus():
-    """Detects AMD GPUs via rocm-smi or /dev/dri."""
+    """Detect AMD GPUs"""
     try:
-        # Try rocm-smi first
+        result = subprocess.run(
-        res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+            ["rocm-smi", "--showid", "--csv"],
-        if res.returncode == 0:
+            capture_output=True,
-            count = res.stdout.count("GPU")
+            text=True,
-            if count > 0: return count
+            timeout=10
-    except: pass
+        )
+        if result.returncode == 0:
+            count = result.stdout.count("GPU")
+            if count > 0:
+                return count
+    except Exception as e:
+        log(f"Warning: rocm-smi failed: {e}")
     
-    # Fallback to /dev/dri/render*
+    # Fallback to /dev/dri
     try:
-        return len(list(Path("/dev/dri").glob("renderD*")))
+        render_devices = list(Path("/dev/dri").glob("renderD*"))
-    except:
+        if render_devices:
+            return len(render_devices)
+    except Exception:
+        pass
+    
+    log("Warning: Could not detect GPUs, assuming 1 GPU")
     return 1
 
-def get_verified_config(model_id, tp_size, max_seqs):
+def find_model_path(model_id):
-    """
+    """Find local model path"""
-    Reads max_context_results.json to find the best verified configuration.
+    log(f"Looking for model: {model_id}")
-    Returns dict: {'ctx': int, 'util': float}
+    log(f"LOCAL_MODEL_DIR: {LOCAL_MODEL_DIR}")
-    """
-    default_config = {
-        "ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)),
-        "util": 0.90 # Safe default
-    }
     
-    if not RESULTS_FILE.exists():
+    if not os.path.exists(LOCAL_MODEL_DIR):
-        return default_config
+        log(f"ERROR: LOCAL_MODEL_DIR does not exist: {LOCAL_MODEL_DIR}")
+        return None
     
-    try:
+    # Try exact match
-        with open(RESULTS_FILE, "r") as f:
+    candidate = os.path.join(LOCAL_MODEL_DIR, model_id)
-            data = json.load(f)
+    if os.path.isdir(candidate):
+        log(f"Found model at: {candidate}")
+        return candidate
     
-        # Filter for Model + TP + Sequences
+    # Try without prefix
-        matches = [r for r in data 
+    repo_name = model_id.split('/')[-1] if '/' in model_id else model_id
-                  if r["model"] == model_id 
+    candidate = os.path.join(LOCAL_MODEL_DIR, repo_name)
-                  and r["tp"] == tp_size 
+    if os.path.isdir(candidate):
-                  and r["max_seqs"] == max_seqs 
+        log(f"Found model at: {candidate}")
-                  and r["status"] == "success"]
+        return candidate
     
-        if not matches:
+    # Try case-insensitive match
-            # Fallback 1: Try finding match with SAME TP but ANY Sequences (e.g. 1) to get base context?
-            # Actually, safer to fallback to default or try finding nearest sequence?
-            # Let's try finding exact match first. If fail, return default.
-            return default_config
-            
-        # Sort by Util desc, then Context desc
-        # We prefer higher utilization if available (performance), as long as it is verified success
-        matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True)
-        
-        best = matches[0]
-        return {
-            "ctx": best["max_context_1_user"],
-            "util": float(best["util"])
-        }
-        
-    except Exception as e:
-        return default_config
-
-def run_dialog(args):
-    """Runs dialog and returns stderr (selection)."""
-    with tempfile.NamedTemporaryFile(mode="w+") as tf:
-        cmd = ["dialog"] + args
-        try:
-            subprocess.run(cmd, stderr=tf, check=True)
-            tf.seek(0)
-            return tf.read().strip()
-        except subprocess.CalledProcessError:
-            return None # User cancelled
-
-def nuke_vllm_cache():
-    """Removes vLLM cache directory to fix potential graph/incompatibility issues."""
-    cache = Path.home() / ".cache" / "vllm"
-    if cache.exists():
-        try:
-            print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
-            subprocess.run(["rm", "-rf", str(cache)], check=True)
-            cache.mkdir(parents=True, exist_ok=True)
-            print(" Done.")
-            time.sleep(1)
-        except Exception as e:
-            print(f" Failed: {e}")
-
-def configure_and_launch(model_idx, gpu_count):
-    model_id = MODELS_TO_RUN[model_idx]
-    config = MODEL_TABLE[model_id]
-    
-    # Determine whether we have a local copy to serve. Try multiple fallbacks:
-    # 1) LOCAL_MODEL_DIR/<owner>/<repo>
-    # 2) LOCAL_MODEL_DIR/<repo>
-    # 3) case-insensitive match of <repo> in LOCAL_MODEL_DIR
-    model_path = model_id
-    if LOCAL_MODEL_DIR:
-        # Full repo path (owner/repo)
-        candidate_full = os.path.join(LOCAL_MODEL_DIR, model_id)
-        if os.path.isdir(candidate_full):
-            model_path = candidate_full
-        else:
-            # Repo-name only (last segment)
-            repo_name = model_id.split('/')[-1]
-            candidate_repo = os.path.join(LOCAL_MODEL_DIR, repo_name)
-            if os.path.isdir(candidate_repo):
-                model_path = candidate_repo
-            else:
-                # Fallback: try to find a directory in LOCAL_MODEL_DIR that matches repo_name case-insensitively
     try:
         for entry in os.listdir(LOCAL_MODEL_DIR):
             if entry.lower() == repo_name.lower():
                 entry_path = os.path.join(LOCAL_MODEL_DIR, entry)
                 if os.path.isdir(entry_path):
-                                model_path = entry_path
+                    log(f"Found model at: {entry_path}")
-                                break
+                    return entry_path
-                except Exception:
+    except Exception as e:
-                    pass
+        log(f"ERROR: Failed to list directory: {e}")
-+    # if LOCAL_MODEL_DIR is specified, refuse to fall back to remote
-+    if LOCAL_MODEL_DIR and model_path == model_id:
-+        print(f"Error: model '{model_id}' not found under LOCAL_MODEL_DIR={LOCAL_MODEL_DIR}")
-+        print("Off‑line mode active; network downloads are disabled.")
-+        sys.exit(1)
     
-    # Static Config
+    log(f"ERROR: Model not found: {model_id}")
-    valid_tps = config.get("valid_tp", [1])
+    log(f"Available models: {os.listdir(LOCAL_MODEL_DIR)}")
-    max_tp = max(valid_tps) if valid_tps else 1
+    return None
 
-    # Defaults
+def launch_model(model_id, config, model_path, gpu_count):
-    current_tp = min(gpu_count, max_tp)
+    """Launch vLLM server"""
-    current_seqs = 1 # Default to 1 concurrent user/request for stability
+    log(f"Launching model: {model_id}")
     
-    # Initial Lookup
+    # Get configuration
-    verified = get_verified_config(model_id, current_tp, current_seqs)
+    valid_tp = config.get("valid_tp", [1])
-    current_ctx = verified["ctx"]
+    max_tp = max(valid_tp) if valid_tp else 1
-    current_util = verified["util"]
     
-    clear_cache = False
+    # Check for manual TP_SIZE override
-    use_eager = config.get("enforce_eager", False) # Default to model config, usually False
+    tp_size_env = os.getenv("TP_SIZE")
-    use_rocm_attn = False # Default to Triton
+    if tp_size_env:
-    
+        tp_size = int(tp_size_env)
-    name = model_id.split("/")[-1]
+        log(f"TP_SIZE environment variable set: {tp_size}")
-    
+        if tp_size not in valid_tp:
-    while True:
+            log(f"WARNING: TP_SIZE={tp_size} is not in valid_tp={valid_tp}, proceeding anyway")
-        cache_status = "YES" if clear_cache else "NO"
-        eager_status = "YES" if use_eager else "NO"
-        attn_backend = "ROCm" if use_rocm_attn else "Triton"
-        
-        menu_args = [
-            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
-            "--title", f"Configuration: {name}",
-            "--menu", "Customize Launch Parameters:", "22", "65", "9",
-            "1", f"Tensor Parallelism:   {current_tp}",
-            "2", f"Concurrent Requests:  {current_seqs}",
-            "3", f"Context Length:       {current_ctx} (Verified)",
-            "4", f"GPU Utilization:      {current_util} (Verified)",
-            "5", f"Attention Backend:    {attn_backend}",
-            "6", f"Erase vLLM Cache:     {cache_status}",
-            "7", f"Force Eager Mode:     {eager_status}",
-            "8", "LAUNCH SERVER"
-        ]
-        
-        choice = run_dialog(menu_args)
-        if not choice: return False # Back/Cancel
-        
-        if choice == "1":
-            # TP Selection
-            new_tp = run_dialog([
-                "--title", "Tensor Parallelism",
-                "--rangebox", f"Set TP Size (1-{max_tp})", "10", "40", "1", str(max_tp), str(current_tp)
-            ])
-            if new_tp: 
-                new_tp_int = int(new_tp)
-                if new_tp_int != current_tp:
-                    current_tp = new_tp_int
-                    # RE-CALCULATE Config
-                    verified = get_verified_config(model_id, current_tp, current_seqs)
-                    current_ctx = verified["ctx"]
-                    current_util = verified["util"]
-            
-        elif choice == "2":
-            # Max Seqs Selection
-            new_seqs = run_dialog([
-                "--title", "Concurrent Requests",
-                "--menu", "Select Max Concurrent Requests:", "12", "40", "4",
-                "1", "1 (Latency Focus)",
-                "4", "4 (Balanced)",
-                "8", "8 (Throughput)",
-                "16", "16 (Max Load)"
-            ])
-            if new_seqs:
-                current_seqs = int(new_seqs)
-                # RE-CALCULATE Config based on new concurrency
-                verified = get_verified_config(model_id, current_tp, current_seqs)
-                current_ctx = verified["ctx"]
-                current_util = verified["util"]
-
-        elif choice == "3":
-            # Configured Length Override
-            new_ctx = run_dialog([
-                "--title", "Context Length",
-                "--inputbox", f"Override verified limit ({current_ctx}):", "10", "40", str(current_ctx)
-            ])
-            if new_ctx: current_ctx = int(new_ctx)
-
-        elif choice == "4":
-             # Util Override
-             pass 
-
-        elif choice == "5":
-            # Toggle Attention Backend
-            use_rocm_attn = not use_rocm_attn
-
-        elif choice == "6":
-            # Toggle Cache
-            if not clear_cache:
-                # Enabling it -> Show Warning
-                warn_msg = (
-                    "WARNING: Erasing the vLLM cache will remove the compiled compute graphs.\n\n"
-                    "This is useful if you are experiencing crashes, 'invalid graph' errors,\n"
-                    "or have switched vLLM versions recently.\n\n"
-                    "However, the next startup will take longer as graphs are re-compiled.\n\n"
-                    "Are you sure you want to enable this?"
-                )
-                confirm = run_dialog([
-                    "--title", "Erase Cache Warning", 
-                    "--yesno", warn_msg, "12", "60"
-                ])
-                
-                # If confirm is not None (exit 0), it is YES.
-                if confirm is not None:
-                     clear_cache = True
     else:
-                # Disabling it -> No warning needed
+        tp_size = min(gpu_count, max_tp)
-                clear_cache = False
     
-        elif choice == "7":
+    ctx = int(config.get("ctx", 8192))
-            # Toggle Eager Mode
+    max_seqs = int(config.get("max_num_seqs", 64))
-            use_eager = not use_eager
+    gpu_util = float(config.get("gpu_util", 0.98))
+    served_model_name = config.get("served_model_name", model_id)
     
-        elif choice == "8":
+    log(f"Config: TP={tp_size}, Ctx={ctx}, Seqs={max_seqs}, Util={gpu_util}")
-            # Launch
-            break
-            
-    # Build Command
-    subprocess.run(["clear"])
-    
-    if clear_cache:
-        nuke_vllm_cache()
     
+    # Build command
     cmd = [
         "vllm", "serve", model_path,
+        "--served-model-name", served_model_name,
         "--host", HOST,
         "--port", PORT,
-        "--tensor-parallel-size", str(current_tp),
+        "--tensor-parallel-size", str(tp_size),
-        "--max-num-seqs", str(current_seqs),
+        "--max-num-seqs", str(max_seqs),
-        "--max-model-len", str(current_ctx),
+        "--max-model-len", str(ctx),
-        "--gpu-memory-utilization", str(current_util),
+        "--gpu-memory-utilization", str(gpu_util),
         "--dtype", "auto"
     ]
     
-    if config.get("trust_remote"): cmd.append("--trust-remote-code")
+    if config.get("trust_remote"):
-    if use_eager: cmd.append("--enforce-eager")
+        cmd.append("--trust-remote-code")
     
-    # Env Vars
+    if config.get("enforce_eager"):
+        cmd.append("--enforce-eager")
+    
+    # Add Qwen3.5 specific parameters
+    if "qwen3.5" in model_id.lower() or "qwen3_5" in model_id.lower():
+        cmd.extend(["--quantization", "moe_wna16"])
+        cmd.extend(["--reasoning-parser", "qwen3"])
+        log("Added Qwen3.5 specific parameters: --quantization moe_wna16 --reasoning-parser qwen3")
+    
+    # Add tool call parser if specified
+    tool_call_parser = config.get("tool_call_parser")
+    openclaw_compat = os.getenv("OPENCLAW_COMPAT", "false").lower() == "true"
+    if openclaw_compat and not tool_call_parser:
+        tool_call_parser = os.getenv("OPENCLAW_TOOL_CALL_PARSER", "qwen3_xml")
+    if tool_call_parser:
+        cmd.extend(["--tool-call-parser", tool_call_parser])
+        enable_auto_tool_choice = config.get("enable_auto_tool_choice")
+        auto_tool_choice_env = os.getenv("ENABLE_AUTO_TOOL_CHOICE")
+        if auto_tool_choice_env is not None:
+            enable_auto_tool_choice = auto_tool_choice_env.lower() == "true"
+        if enable_auto_tool_choice is None:
+            enable_auto_tool_choice = True
+        if openclaw_compat:
+            enable_auto_tool_choice = True
+        if enable_auto_tool_choice:
+            cmd.extend(["--enable-auto-tool-choice"])
+            log("Added auto tool choice enabled")
+        else:
+            log("Auto tool choice disabled")
+        log(f"Added tool call parser: {tool_call_parser}")
+    
+    log(f"Command: {' '.join(cmd)}")
+    
+    # Set environment
     env = os.environ.copy()
     env.update(config.get("env", {}))
     
-    if use_rocm_attn:
+    # Add AMD GPU tensor parallel environment variables
-        env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
+    if tp_size > 1:
-        env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
+        env["NCCL_DEBUG"] = "INFO"
-        # Optional: Explicitly mention these in print
+        env["NCCL_SOCKET_IFNAME"] = "^lo,docker0"
+        env["NCCL_P2P_DISABLE"] = "0"
+        env["NCCL_SHM_DISABLE"] = "0"
+        env["NCCL_IB_DISABLE"] = "1"
+        env["NCCL_P2P_LEVEL"] = "SYS"
+        env["HIP_VISIBLE_DEVICES"] = os.getenv("HIP_VISIBLE_DEVICES", "0,1")
+        log("Added NCCL environment variables for AMD GPU tensor parallel")
+        log(f"HIP_VISIBLE_DEVICES: {env['HIP_VISIBLE_DEVICES']}")
     
-    
+    # Launch vLLM
-    print("\n" + "="*60)
+    log("Starting vLLM server...")
-    print(f" Launching: {name}")
+    try:
-    if model_path != model_id:
+        result = subprocess.run(cmd, env=env)
-        print(f" (using local model at {model_path})")
+        if result.returncode != 0:
-    print(f" Config:    TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx} | Util={current_util}")
+            log(f"ERROR: vLLM exited with code {result.returncode}")
-    print(f" Backend:   {'ROCm' if use_rocm_attn else 'Triton'}")
+            sys.exit(result.returncode)
-    if clear_cache:
+    except Exception as e:
-        print(f" Action:    Clearing vLLM Cache (~/.cache/vllm)")
+        log(f"ERROR: Failed to start vLLM: {e}")
-    print(f" Command:   {' '.join(cmd)}")
+        import traceback
-    print("="*60 + "\n")
+        traceback.print_exc()
-    
+        sys.exit(1)
-    os.execvpe("vllm", cmd, env)
 
 def main():
-    check_dependencies()
+    """Main entry point"""
+    log("Starting vLLM launcher...")
+    
+    # Load configuration
+    model_table, default_model, models_to_run = load_config()
+    
+    # Detect GPUs
     gpu_count = detect_gpus()
+    log(f"Detected {gpu_count} GPU(s)")
     
-    while True:
+    # Check if we should use default model
-        # Build Model Menu
+    use_default = os.getenv("USE_DEFAULT_MODEL", "false").lower() == "true"
-        menu_items = []
-        for i, m_id in enumerate(MODELS_TO_RUN):
-            name = m_id.split("/")[-1]
-            # Pre-calc verified ctx for 'default' TP to show in menu? 
-            # Or just show names. Just names is cleaner.
-            config = MODEL_TABLE[m_id]
-            menu_items.extend([str(i), name])
     
-        choice = run_dialog([
+    if use_default:
-            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+        log("Using default model mode")
-            "--title", "Select Model",
+        model_id = default_model
-            "--menu", "Choose a model to serve:", "20", "60", "10"
+    else:
-        ] + menu_items)
+        # Interactive mode - for now just use default
+        log("Interactive mode not supported in container, using default model")
+        model_id = default_model
     
-        if not choice:
+    # Check if model is in config
-            subprocess.run(["clear"])
+    if model_id not in model_table:
-            print("Selection cancelled.")
+        log(f"ERROR: Model {model_id} not found in config")
-            sys.exit(0)
+        sys.exit(1)
     
-        configure_and_launch(int(choice), gpu_count)
+    config = model_table[model_id]
+    
+    # Find model path
+    model_path = find_model_path(model_id)
+    if not model_path:
+        log("ERROR: Could not find local model. Offline mode is active.")
+        sys.exit(1)
+    
+    # Launch model
+    launch_model(model_id, config, model_path, gpu_count)
 
 if __name__ == "__main__":
     main()

scripts/test_start_vllm.py

@@ -0,0 +1,317 @@
+cat start-vllm
+#!/usr/bin/env python3
+import sys
+import os
+import json
+import shutil
+import tempfile
+import subprocess
+from pathlib import Path
+
+# Add benchmarks dir to path to import config
+# Add benchmarks dir to path to import config
+SCRIPT_DIR = Path(__file__).parent.resolve()
+BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
+OPT_DIR = Path("/opt")
+
+# Check /opt first (Container), then local fallback
+if (OPT_DIR / "run_vllm_bench.py").exists():
+    sys.path.append(str(OPT_DIR))
+else:
+    sys.path.append(str(BENCH_DIR))
+
+try:
+    from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
+except ImportError:
+    print("Error: Could not import run_vllm_bench.py config.")
+    sys.exit(1)
+
+if (OPT_DIR / "max_context_results.json").exists():
+    RESULTS_FILE = OPT_DIR / "max_context_results.json"
+else:
+    RESULTS_FILE = BENCH_DIR / "max_context_results.json"
+HOST = os.getenv("HOST", "0.0.0.0")
+PORT = os.getenv("PORT", "8000")
+
+def check_dependencies():
+    if not shutil.which("dialog"):
+        print("Error: 'dialog' is required. Please install it (apt-get install dialog).")
+        sys.exit(1)
+
+def detect_gpus():
+    """Detects AMD GPUs via rocm-smi or /dev/dri."""
+    try:
+        # Try rocm-smi first
+        res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if res.returncode == 0:
+            count = res.stdout.count("GPU")
+            if count > 0: return count
+    except: pass
+    
+    # Fallback to /dev/dri/render*
+    try:
+        return len(list(Path("/dev/dri").glob("renderD*")))
+    except:
+        return 1
+
+def get_verified_config(model_id, tp_size, max_seqs):
+    """
+    Reads max_context_results.json to find the best verified configuration.
+    Returns dict: {'ctx': int, 'util': float}
+    """
+    default_config = {
+        "ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)),
+        "util": 0.90 # Safe default
+    }
+    
+    if not RESULTS_FILE.exists():
+        return default_config
+
+    try:
+        with open(RESULTS_FILE, "r") as f:
+            data = json.load(f)
+            
+        # Filter for Model + TP + Sequences
+        matches = [r for r in data 
+                  if r["model"] == model_id 
+                  and r["tp"] == tp_size 
+                  and r["max_seqs"] == max_seqs 
+                  and r["status"] == "success"]
+        
+        if not matches:
+            # Fallback 1: Try finding match with SAME TP but ANY Sequences (e.g. 1) to get base context?
+            # Actually, safer to fallback to default or try finding nearest sequence?
+            # Let's try finding exact match first. If fail, return default.
+            return default_config
+            
+        # Sort by Util desc, then Context desc
+        # We prefer higher utilization if available (performance), as long as it is verified success
+        matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True)
+        
+        best = matches[0]
+        return {
+            "ctx": best["max_context_1_user"],
+            "util": float(best["util"])
+        }
+        
+    except Exception as e:
+        return default_config
+
+def run_dialog(args):
+    """Runs dialog and returns stderr (selection)."""
+    with tempfile.NamedTemporaryFile(mode="w+") as tf:
+        cmd = ["dialog"] + args
+        try:
+            subprocess.run(cmd, stderr=tf, check=True)
+            tf.seek(0)
+            return tf.read().strip()
+        except subprocess.CalledProcessError:
+            return None # User cancelled
+
+def nuke_vllm_cache():
+    """Removes vLLM cache directory to fix potential graph/incompatibility issues."""
+    cache = Path.home() / ".cache" / "vllm"
+    if cache.exists():
+        try:
+            print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
+            subprocess.run(["rm", "-rf", str(cache)], check=True)
+            cache.mkdir(parents=True, exist_ok=True)
+            print(" Done.")
+            time.sleep(1)
+        except Exception as e:
+            print(f" Failed: {e}")
+
+def configure_and_launch(model_idx, gpu_count):
+    model_id = MODELS_TO_RUN[model_idx]
+    config = MODEL_TABLE[model_id]
+    
+    # Static Config
+    valid_tps = config.get("valid_tp", [1])
+    max_tp = max(valid_tps) if valid_tps else 1
+    
+    # Defaults
+    current_tp = min(gpu_count, max_tp)
+    current_seqs = 1 # Default to 1 concurrent user/request for stability
+    
+    # Initial Lookup
+    verified = get_verified_config(model_id, current_tp, current_seqs)
+    current_ctx = verified["ctx"]
+    current_util = verified["util"]
+    
+    clear_cache = False
+    use_eager = config.get("enforce_eager", False) # Default to model config, usually False
+    use_rocm_attn = False # Default to Triton
+    
+    name = model_id.split("/")[-1]
+    
+    while True:
+        cache_status = "YES" if clear_cache else "NO"
+        eager_status = "YES" if use_eager else "NO"
+        attn_backend = "ROCm" if use_rocm_attn else "Triton"
+        
+        menu_args = [
+            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+            "--title", f"Configuration: {name}",
+            "--menu", "Customize Launch Parameters:", "22", "65", "9",
+            "1", f"Tensor Parallelism:   {current_tp}",
+            "2", f"Concurrent Requests:  {current_seqs}",
+            "3", f"Context Length:       {current_ctx} (Verified)",
+            "4", f"GPU Utilization:      {current_util} (Verified)",
+            "5", f"Attention Backend:    {attn_backend}",
+            "6", f"Erase vLLM Cache:     {cache_status}",
+            "7", f"Force Eager Mode:     {eager_status}",
+            "8", "LAUNCH SERVER"
+        ]
+        
+        choice = run_dialog(menu_args)
+        if not choice: return False # Back/Cancel
+        
+        if choice == "1":
+            # TP Selection
+            new_tp = run_dialog([
+                "--title", "Tensor Parallelism",
+                "--rangebox", f"Set TP Size (1-{max_tp})", "10", "40", "1", str(max_tp), str(current_tp)
+            ])
+            if new_tp: 
+                new_tp_int = int(new_tp)
+                if new_tp_int != current_tp:
+                    current_tp = new_tp_int
+                    # RE-CALCULATE Config
+                    verified = get_verified_config(model_id, current_tp, current_seqs)
+                    current_ctx = verified["ctx"]
+                    current_util = verified["util"]
+            
+        elif choice == "2":
+            # Max Seqs Selection
+            new_seqs = run_dialog([
+                "--title", "Concurrent Requests",
+                "--menu", "Select Max Concurrent Requests:", "12", "40", "4",
+                "1", "1 (Latency Focus)",
+                "4", "4 (Balanced)",
+                "8", "8 (Throughput)",
+                "16", "16 (Max Load)"
+            ])
+            if new_seqs:
+                current_seqs = int(new_seqs)
+                # RE-CALCULATE Config based on new concurrency
+                verified = get_verified_config(model_id, current_tp, current_seqs)
+                current_ctx = verified["ctx"]
+                current_util = verified["util"]
+
+        elif choice == "3":
+            # Configured Length Override
+            new_ctx = run_dialog([
+                "--title", "Context Length",
+                "--inputbox", f"Override verified limit ({current_ctx}):", "10", "40", str(current_ctx)
+            ])
+            if new_ctx: current_ctx = int(new_ctx)
+
+        elif choice == "4":
+             # Util Override
+             pass 
+
+        elif choice == "5":
+            # Toggle Attention Backend
+            use_rocm_attn = not use_rocm_attn
+
+        elif choice == "6":
+            # Toggle Cache
+            if not clear_cache:
+                # Enabling it -> Show Warning
+                warn_msg = (
+                    "WARNING: Erasing the vLLM cache will remove the compiled compute graphs.\n\n"
+                    "This is useful if you are experiencing crashes, 'invalid graph' errors,\n"
+                    "or have switched vLLM versions recently.\n\n"
+                    "However, the next startup will take longer as graphs are re-compiled.\n\n"
+                    "Are you sure you want to enable this?"
+                )
+                confirm = run_dialog([
+                    "--title", "Erase Cache Warning", 
+                    "--yesno", warn_msg, "12", "60"
+                ])
+                
+                # If confirm is not None (exit 0), it is YES.
+                if confirm is not None:
+                     clear_cache = True
+            else:
+                # Disabling it -> No warning needed
+                clear_cache = False
+             
+        elif choice == "7":
+            # Toggle Eager Mode
+            use_eager = not use_eager
+             
+        elif choice == "8":
+            # Launch
+            break
+            
+    # Build Command
+    subprocess.run(["clear"])
+    
+    if clear_cache:
+        nuke_vllm_cache()
+    
+    cmd = [
+        "vllm", "serve", model_id,
+        "--host", HOST,
+        "--port", PORT,
+        "--tensor-parallel-size", str(current_tp),
+        "--max-num-seqs", str(current_seqs),
+        "--max-model-len", str(current_ctx),
+        "--gpu-memory-utilization", str(current_util),
+        "--dtype", "auto"
+    ]
+    
+    if config.get("trust_remote"): cmd.append("--trust-remote-code")
+    if use_eager: cmd.append("--enforce-eager")
+    
+    # Env Vars
+    env = os.environ.copy()
+    env.update(config.get("env", {}))
+    
+    if use_rocm_attn:
+        env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
+        env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
+        # Optional: Explicitly mention these in print
+        
+    
+    print("\n" + "="*60)
+    print(f" Launching: {name}")
+    print(f" Config:    TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx} | Util={current_util}")
+    print(f" Backend:   {'ROCm' if use_rocm_attn else 'Triton'}")
+    if clear_cache:
+        print(f" Action:    Clearing vLLM Cache (~/.cache/vllm)")
+    print(f" Command:   {' '.join(cmd)}")
+    print("="*60 + "\n")
+    
+    os.execvpe("vllm", cmd, env)
+
+def main():
+    check_dependencies()
+    gpu_count = detect_gpus()
+    
+    while True:
+        # Build Model Menu
+        menu_items = []
+        for i, m_id in enumerate(MODELS_TO_RUN):
+            name = m_id.split("/")[-1]
+            # Pre-calc verified ctx for 'default' TP to show in menu? 
+            # Or just show names. Just names is cleaner.
+            config = MODEL_TABLE[m_id]
+            menu_items.extend([str(i), name])
+            
+        choice = run_dialog([
+            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+            "--title", "Select Model",
+            "--menu", "Choose a model to serve:", "20", "60", "10"
+        ] + menu_items)
+        
+        if not choice:
+            subprocess.run(["clear"])
+            print("Selection cancelled.")
+            sys.exit(0)
+            
+        configure_and_launch(int(choice), gpu_count)
+
+if __name__ == "__main__":
+    main()

test.py

@@ -0,0 +1,19 @@
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://192.168.0.11:8000/v1",
+    api_key="dummy"  # vLLM 不需要真实的 API key
+)
+
+# 聊天完成
+response = client.chat.completions.create(
+    model="Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "你好，请介绍一下你自己"}
+    ],
+    max_tokens=500,
+    temperature=0.7
+)
+
+print(response.choices[0].message.content)