x

Dockerfile

@@ -1,180 +1,27 @@
-FROM registry.fedoraproject.org/fedora:43
+FROM docker.1ms.run/kyuz0/vllm-therock-gfx1201:latest
 
-# 1. System Base & Build Tools
+# Set environment variables
-# Added 'gperftools-libs' for tcmalloc (fixes double-free)
-RUN dnf -y --refresh --setopt=install_weak_deps=False --setopt=metadata_expire=3600 --setopt=timeout=600 install --nodocs \
-  python3.13 python3.13-devel git bash ca-certificates curl \
-  gcc gcc-c++ binutils make cmake ninja-build \
-  libdrm-devel zlib-devel openssl-devel \
-  numactl-devel gperftools-libs dialog procps-ng \
-  && dnf clean all && rm -rf /var/cache/dnf/*
-
-# Install additional tools in a separate step
-RUN dnf -y --refresh --setopt=install_weak_deps=False --setopt=metadata_expire=3600 --setopt=timeout=600 install --nodocs \
-  rsync libatomic ffmpeg-free aria2c tar xz vim nano jq \
-  && dnf clean all && rm -rf /var/cache/dnf/*
-
-# 2. Install "TheRock" ROCm SDK (Tarball Method)
-# Note: You can pre-download the tarball to /tmp/therock.tar.gz to speed up build
-WORKDIR /tmp
-ARG ROCM_MAJOR_VER=7
-ARG GFX=gfx120X-all
-RUN if [ -f /tmp/therock.tar.gz ]; then \
-    echo "Using pre-downloaded tarball"; \
-  else \
-    set -euo pipefail; \
-    BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
-    PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
-    KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
-    | tr '<' '\n' \
-    | grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
-    | sort -V | tail -n1)"; \
-    echo "Downloading Latest Tarball: ${KEY}"; \
-    aria2c -x 16 -s 16 -j 16 --file-allocation=none --async-dns=false \
-      --max-overall-download-limit=0 --max-download-limit=0 \
-      --min-split-size=1M --retry-wait=5 --max-tries=0 \
-      --timeout=600 --connect-timeout=60 \
-      "${BASE}/${KEY}" -o therock.tar.gz; \
-  fi && \
-  mkdir -p /opt/rocm && \
-  tar xzf therock.tar.gz -C /opt/rocm --strip-components=1 && \
-  rm therock.tar.gz
-
-# 3. Configure Global ROCm Environment
-# We add LD_PRELOAD for tcmalloc here to fix the shutdown crash
-RUN export ROCM_PATH=/opt/rocm && \
-  BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
-  printf '%s\n' \
-  "export ROCM_PATH=/opt/rocm" \
-  "export HIP_PLATFORM=amd" \
-  "export HIP_PATH=/opt/rocm" \
-  "export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \
-  "export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \
-  "export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \
-  "export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \
-  "export ROCBLAS_USE_HIPBLASLT=1" \
-  "export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \
-  "export VLLM_TARGET_DEVICE=rocm" \
-  "export HIP_FORCE_DEV_KERNARG=1" \
-  "export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \
-  "export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \
-  > /etc/profile.d/rocm-sdk.sh && \
-  chmod 0644 /etc/profile.d/rocm-sdk.sh
-
-# 4. Python Venv Setup
-RUN /usr/bin/python3.13 -m venv /opt/venv
-ENV VIRTUAL_ENV=/opt/venv
-ENV PATH=/opt/venv/bin:$PATH
-ENV PIP_NO_CACHE_DIR=1
-RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh
-RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0"
-
-# 5. Install PyTorch (TheRock Nightly)
-RUN python -m pip install \
-  --index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \
-  --pre torch torchaudio torchvision
-
-# Flash-Attention
-WORKDIR /opt
-ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
-
-RUN git clone https://github.com/ROCm/flash-attention.git &&\ 
-  cd flash-attention &&\
-  git checkout main_perf &&\
-  python setup.py install && \
-  cd /opt && rm -rf /opt/flash-attention
-
-# 6. Clone vLLM
-RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm
-WORKDIR /opt/vllm
-
-# --- PATCHING ---
-# vLLM relies on 'amdsmi' to detect AMD GPUs. If it's missing or fails (common in containers),
-# vLLM falls back to CPU. We patch it to force ROCm detection.
-RUN echo "import sys, re" > patch_vllm.py && \
-  echo "from pathlib import Path" >> patch_vllm.py && \
-  # Patch 1: __init__.py - Force is_rocm=True and bypass amdsmi checks
-  echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \
-  echo "txt = p.read_text()" >> patch_vllm.py && \
-  echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \
-  echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \
-  echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \
-  echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \
-  echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \
-  echo "p.write_text(txt)" >> patch_vllm.py && \
-  # Patch 2: rocm.py - Mock amdsmi and force device name
-  echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \
-  echo "txt = p.read_text()" >> patch_vllm.py && \
-  echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \
-  echo "txt = header + txt" >> patch_vllm.py && \
-  echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \
-  echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \
-  echo "txt += '\n    def get_device_name(self, device_id: int = 0) -> str:\n        return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \
-  echo "p.write_text(txt)" >> patch_vllm.py && \
-  echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \
-  python patch_vllm.py
-
-# 7. Build vLLM (Wheel Method) with CLANG Host Compiler
-RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11
-ENV ROCM_HOME="/opt/rocm"
-ENV HIP_PATH="/opt/rocm"
-ENV VLLM_TARGET_DEVICE="rocm"
-ENV PYTORCH_ROCM_ARCH="gfx1201"
-ENV HIP_ARCHITECTURES="gfx1201"          
-ENV AMDGPU_TARGETS="gfx1201"              
-ENV MAX_JOBS="32"
-
-# --- FIX FOR SEGFAULT ---
-# We force the Host Compiler (CC/CXX) to be the ROCm Clang, not Fedora GCC.
-# This aligns the ABI of the compiled vLLM extensions with PyTorch.
-ENV CC="/opt/rocm/llvm/bin/clang"
-ENV CXX="/opt/rocm/llvm/bin/clang++"
-
-RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
-  echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \
-  export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
-  export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
-  python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \
-  python -m pip install /tmp/dist/*.whl
-
-# --- bitsandbytes (ROCm) ---
-WORKDIR /opt
-RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git
-WORKDIR /opt/bitsandbytes
-
-# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile)
-ENV HIP_PLATFORM="amd"
-ENV CMAKE_PREFIX_PATH="/opt/rocm"
-
-# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++)
-RUN cmake -S . \
-  -DGPU_TARGETS="gfx1201" \
-  -DBNB_ROCM_ARCH="gfx1201" \
-  -DCOMPUTE_BACKEND=hip \
-  -DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
-  -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
-  && \
-  make -j32 && \
-  python -m pip install --no-cache-dir . --no-build-isolation --no-deps
-
-# 8. Final Cleanup & Runtime
-WORKDIR /opt
-RUN chmod -R a+rwX /opt && \
-  find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \
-  find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \
-  rm -rf /root/.cache/pip || true && \
-  dnf clean all && rm -rf /var/cache/dnf/*
-
-COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh
-COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh
-COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh
-COPY scripts/start_vllm.py /usr/local/bin/start-vllm
-COPY benchmarks/max_context_results.json /opt/max_context_results.json
-COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py
-RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json
-RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
-
-# Set environment variable for default model usage
 ENV USE_DEFAULT_MODEL=true
+ENV LOCAL_MODEL_DIR=/opt/model
 
+# Create necessary directories
+RUN mkdir -p /opt/script /opt/model /config
+
+# Copy configuration file
+COPY config.json /config/config.json
+
+# Copy scripts to /opt/script
+COPY scripts/start_vllm.py /opt/script/start-vllm
+COPY benchmarks/run_vllm_bench.py /opt/script/run_vllm_bench.py
+
+# Make scripts executable
+RUN chmod +x /opt/script/start-vllm
+
+# Create symlink for backward compatibility
+RUN ln -sf /opt/script/start-vllm /usr/local/bin/start-vllm
+
+# Set working directory
+WORKDIR /opt
+
+# Default command
 CMD ["start-vllm"]

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       - USE_DEFAULT_MODEL=true
       - LOCAL_MODEL_DIR=/opt/model
     volumes:
-      - /opt/project/amd-r9700-vllm-toolboxes/config.json:/config.json:ro
+      - /opt/project/amd-r9700-vllm-toolboxes/config.json:/config/config.json:ro
       - /opt/model:/opt/model
     devices:
       - /dev/dri:/dev/dri
@@ -17,4 +17,4 @@ services:
       - render
     security_opt:
       - seccomp=unconfined
-    restart: unless-stopped
+    restart: unless-stopped

scripts/run_vllm_bench.py

@@ -0,0 +1,356 @@
+cat run_vllm_bench.py 
+#!/usr/bin/env python3
+import subprocess, time, json, sys, os, requests, re, argparse
+from pathlib import Path
+
+# =========================
+# ⚙️ GLOBAL SETTINGS
+# =========================
+
+# HARDWARE: 2x AMD Radeon AI PRO R9700 (32GB, RDNA 4)
+GPU_UTIL = "0.98" 
+PORT     = 8000
+HOST     = "127.0.0.1"
+
+# BENCHMARK TOGGLES
+# AITER is disabled/removed.
+ 
+
+# 1. THROUGHPUT CONFIG
+OFF_NUM_PROMPTS      = 1000 
+OFF_FORCED_OUTPUT    = "512"
+# Default fallback if not specified in MODEL_TABLE
+DEFAULT_BATCH_TOKENS = "8192"
+
+# 2. LATENCY CONFIG
+SRV_DURATION    = 180    
+QPS_SWEEP       = [1.0, 4.0] 
+
+# Fallbacks
+FALLBACK_INPUT_LEN  = 1024
+FALLBACK_OUTPUT_LEN = 512
+
+RESULTS_DIR = Path("benchmark_results")
+RESULTS_DIR.mkdir(exist_ok=True)
+
+# =========================
+# 🛠️ MODEL CONFIGURATION 🛠️
+# =========================
+
+MODEL_TABLE = {
+    # 1. Llama 3.1 8B Instruct
+    # MAD uses 131k tokens. We scale to 32k for 32GB VRAM safety.
+    "meta-llama/Meta-Llama-3.1-8B-Instruct": {
+        "ctx": "65536",  
+        "trust_remote": False,
+        "valid_tp": [1, 2],
+        "max_num_seqs": "64",
+        "max_tokens": "32768" 
+    },
+    
+    # 2. GPT-OSS 20B (MXFP4)
+    # MAD Row 0 uses 8192. We match this exactly.
+    "openai/gpt-oss-20b": {
+        "ctx": "32768", 
+        "trust_remote": True,
+        "valid_tp": [1, 2],
+        "max_num_seqs": "64",
+        "max_tokens": "8192"
+    },
+
+    # 3. Qwen 14B FP8
+    # MAD uses 40k. We use 32k.
+    "RedHatAI/Qwen3-14B-FP8-dynamic": {
+        "ctx": "32768", 
+        "trust_remote": True,
+        "valid_tp": [1],
+        "max_num_seqs": "64",
+        "max_tokens": "32768"
+    },
+
+    # 4. Qwen 30B 4-bit
+    "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
+        "ctx": "24576", 
+        "trust_remote": True,
+        "valid_tp": [1, 2],
+        "max_num_seqs": "64",
+        "max_tokens": "32768"
+    },
+
+    # 5. Qwen 80B AWQ (The Big One) [NEW]
+    # Size: ~48GB. Fits on 2x32GB (64GB). Leftover for Cache: ~16GB.
+    # Config: 20k ctx fits in that cache. Eager mode required for stability.
+    "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
+        "ctx": "20480", 
+        "trust_remote": True,
+        "valid_tp": [2], # Too big for single GPU
+        "max_num_seqs": "32", # Lower concurrency for safety
+        "max_tokens": "16384", # Lower batch size because Eager mode is CPU intensive
+        "enforce_eager": False, 
+        "env": {"VLLM_USE_TRITON_AWQ": "1"} # Fixes "Unsupported Hardware" error
+    },
+
+    # 76 Gemma 3 27B FP8
+    "RedHatAI/gemma-3-27b-it-FP8-dynamic": {
+        "ctx": "29000",
+        "trust_remote": True,
+        "valid_tp": [2],
+        "max_num_seqs": "32",
+        "max_tokens": "29000",
+        "gpu_util": "0.94",
+    },
+
+    # 7. Gemma 3 12B FP8
+    "RedHatAI/gemma-3-12b-it-FP8-dynamic": {
+        "ctx": "9900",
+        "trust_remote": True,
+        "valid_tp": [1, 2],
+        "max_num_seqs": "64",
+        "max_tokens": "9900",
+    },
+}
+
+MODELS_TO_RUN = [
+    "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "openai/gpt-oss-20b",
+    "RedHatAI/Qwen3-14B-FP8-dynamic",
+    "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
+    "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
+    "RedHatAI/gemma-3-27b-it-FP8-dynamic",
+    "RedHatAI/gemma-3-12b-it-FP8-dynamic",
+]
+
+# =========================
+# UTILS
+# =========================
+
+def log(msg): print(f"\n[BENCH] {msg}")
+
+def get_gpu_count():
+    try:
+        # Using rocm-smi --showid to list GPUs. 
+        # Output format: "GPU[0] : Device Name: ..."
+        res = subprocess.run(["rocm-smi", "--showid"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if res.returncode == 0:
+            # Filter specifically for the target GPU as requested
+            target_gpu = "AMD Radeon AI PRO R9700"
+            count = 0
+            for line in res.stdout.strip().split('\n'):
+                if "Device Name" in line and target_gpu in line:
+                    count += 1
+            
+            return count if count > 0 else 1
+        else:
+            log("rocm-smi failed, defaulting to 2 GPUs (Hardcoded Fallback)")
+            return 2
+    except Exception as e:
+        log(f"Error detecting GPUs: {e}, defaulting to 2 GPUs")
+        return 2
+
+def kill_vllm():
+    subprocess.run("pgrep -f 'vllm serve' | xargs -r kill -9", 
+                   shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    time.sleep(5)
+
+def nuke_vllm_cache():
+    cache = Path.home() / ".cache" / "vllm"
+    if cache.exists():
+        try:
+            subprocess.run(["rm", "-rf", str(cache)], check=True)
+            cache.mkdir(parents=True, exist_ok=True)
+            time.sleep(2)
+        except: pass
+
+def get_dataset():
+    data_path = Path("ShareGPT_V3_unfiltered_cleaned_split.json")
+    if data_path.exists(): return str(data_path)
+    
+    log("Downloading ShareGPT dataset...")
+    url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json"
+    try:
+        r = requests.get(url, stream=True, timeout=15)
+        r.raise_for_status()
+        with open(data_path, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192): f.write(chunk)
+        return str(data_path)
+    except Exception as e:
+        log(f"WARNING: ShareGPT download failed ({e}). using RANDOM.")
+        return None
+
+def wait_for_server(url, process, timeout=600):
+    start = time.time()
+    while time.time() - start < timeout:
+        if process.poll() is not None:
+            log(f"CRITICAL: Server died! Ret: {process.returncode}")
+            return False
+        try:
+            if requests.get(f"{url}/v1/models", timeout=2).status_code == 200:
+                log("Server ready. Stabilizing...")
+                time.sleep(5)
+                return True
+        except: pass
+        time.sleep(2)
+    return False
+
+def get_model_args(model, tp_size):
+    config = MODEL_TABLE.get(model, {"ctx": "8192", "max_num_seqs": "32"})
+    
+    # Allow per-model GPU utilization override
+    util = config.get("gpu_util", GPU_UTIL)
+
+    cmd = [
+        "--model", model,
+        "--gpu-memory-utilization", util,
+        "--max-model-len", config["ctx"], 
+        "--dtype", "auto",
+        "--tensor-parallel-size", str(tp_size),
+        "--max-num-seqs", config["max_num_seqs"]
+    ]
+    
+    if config.get("trust_remote"): cmd.append("--trust-remote-code")
+    if config.get("enforce_eager"): cmd.append("--enforce-eager")
+    
+    return cmd
+
+def run_throughput(model, tp_size):
+    if tp_size not in MODEL_TABLE[model]["valid_tp"]: return
+    
+    model_safe = model.replace("/", "_")
+    output_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_throughput.json"
+    
+    if output_file.exists():
+        log(f"SKIP Throughput {model} (TP={tp_size})")
+        return
+
+    dataset_path = get_dataset()
+    dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"]
+    
+    # Retrieve Model-Specific Batch Tokens
+    batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS)
+
+    log(f"START Throughput {model} (TP={tp_size}) [Batch: {batch_tokens}]...")
+    kill_vllm()
+    nuke_vllm_cache()
+
+    cmd = ["vllm", "bench", "throughput"] + get_model_args(model, tp_size)
+    cmd.extend([
+        "--num-prompts", str(OFF_NUM_PROMPTS),
+        "--max-num-batched-tokens", batch_tokens,
+        "--output-len", OFF_FORCED_OUTPUT,
+        "--output-json", str(output_file),
+        "--disable-log-stats"
+    ])
+    cmd.extend(dataset_args)
+
+    # ENV Setup: Global + Model Specific
+    env = os.environ.copy()
+    
+    # Inject model specific env vars (e.g. for AWQ)
+    model_env = MODEL_TABLE[model].get("env", {})
+    env.update(model_env)
+
+    try: 
+        subprocess.run(cmd, check=True, env=env)
+    except: 
+        log(f"ERROR: Throughput failed {model}")
+
+def run_latency(model, tp_size):
+    if tp_size not in MODEL_TABLE[model]["valid_tp"]: return
+    model_safe = model.replace("/", "_")
+
+    if all((RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{q}_latency.json").exists() for q in QPS_SWEEP):
+        return
+
+    dataset_path = get_dataset()
+    log(f"START Server {model} (TP={tp_size})...")
+    kill_vllm()
+    nuke_vllm_cache()
+
+    srv_log = open(RESULTS_DIR / f"{model_safe}_tp{tp_size}_server.log", "w")
+    srv_args = [x for x in get_model_args(model, tp_size) if x != "--model" and x != model]
+    
+    # ENV Setup: Global + Model Specific
+    env = os.environ.copy()
+
+    model_env = MODEL_TABLE[model].get("env", {})
+    env.update(model_env)
+
+    proc = subprocess.Popen(["vllm", "serve", model] + srv_args + ["--host", HOST, "--port", str(PORT)], 
+                            stdout=srv_log, stderr=srv_log, env=env)
+
+    try:
+        if not wait_for_server(f"http://{HOST}:{PORT}", proc): return
+
+        for qps in QPS_SWEEP:
+            out_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{qps}_latency.json"
+            if out_file.exists(): continue
+            
+            log(f"BENCH QPS={qps}...")
+            bench_cmd = [
+                "vllm", "bench", "serve",
+                "--model", model,
+                "--base-url", f"http://{HOST}:{PORT}",
+                "--request-rate", str(qps),
+                "--num-prompts", str(int(max(10, SRV_DURATION * qps))),
+                "--trust-remote-code"
+            ]
+            
+            if dataset_path: bench_cmd.extend(["--dataset-name", "sharegpt", "--dataset-path", dataset_path])
+            else: bench_cmd.extend(["--dataset-name", "random", "--random-input-len", "1024", "--random-output-len", "512"])
+
+            res = subprocess.run(bench_cmd, capture_output=True, text=True, env=env)
+            with open(out_file, "w") as f:
+                f.write(json.dumps({"success": res.returncode==0, "raw_output": res.stdout}, indent=2))
+
+    except Exception as e: log(f"CRASH: {e}")
+    finally:
+        proc.terminate()
+        kill_vllm()
+
+def print_summary(tps):
+    print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}")
+    print("-" * 105)
+    
+    for m in MODELS_TO_RUN:
+        msafe = m.replace("/", "_")
+        for tp in tps:
+            if tp not in MODEL_TABLE[m]["valid_tp"]: continue
+            
+            try: 
+                tdata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_throughput.json").read_text())
+                tok_s = f"{tdata.get('tokens_per_second', 0):.1f}"
+            except: tok_s = "N/A"
+
+            first_row = True
+            for q in QPS_SWEEP:
+                try:
+                    ldata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_qps{q}_latency.json").read_text())
+                    raw = ldata["raw_output"]
+                    ttft = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw).group(1)
+                    tpot = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw).group(1)
+                except: ttft, tpot = "-", "-"
+                
+                name_cell = m.split('/')[-1] if (first_row and q == QPS_SWEEP[0]) else ""
+                
+                print(f"{name_cell:<40} | {tp:<2} | {tok_s:<8} | {q:<4} | {ttft:<6} | {tpot:<6}")
+                first_row = False
+            print("-" * 105)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tp", type=int, nargs="+", default=[1, 2])
+    args = parser.parse_args()
+    
+    gpu_count = get_gpu_count()
+    log(f"Detected {gpu_count} AMD GPU(s)")
+    
+    valid_tp_args = [t for t in args.tp if t <= gpu_count]
+    if not valid_tp_args:
+        log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.")
+        sys.exit(0)
+
+    kill_vllm()
+    for tp in valid_tp_args:
+        for m in MODELS_TO_RUN:
+            run_throughput(m, tp)
+            run_latency(m, tp)

scripts/start_vllm.py

@@ -7,18 +7,17 @@ import tempfile
 import subprocess
 from pathlib import Path
 
-# Add benchmarks dir to path to import config
+# Add script dir to path
-SCRIPT_DIR = Path(__file__).parent.resolve()
+SCRIPT_DIR = Path("/opt/script")
-BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
 OPT_DIR = Path("/opt")
 
 # Config file path (check container path first, then local path)
-CONFIG_PATH = Path("/config.json")
+CONFIG_PATH = Path("/config/config.json")
 if not CONFIG_PATH.exists():
-    CONFIG_PATH = SCRIPT_DIR.parent / "config.json"
+    CONFIG_PATH = Path("/config.json")
 
 # Local model directory (container path)
-LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", str(SCRIPT_DIR.parent / "models"))
+LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", "/opt/model")
 
 # Load configuration from config.json
 try:
@@ -31,10 +30,11 @@ except Exception as e:
     print(f"Error: Could not load config.json: {e}")
     sys.exit(1)
 
+# Results file path
 if (OPT_DIR / "max_context_results.json").exists():
     RESULTS_FILE = OPT_DIR / "max_context_results.json"
 else:
-    RESULTS_FILE = BENCH_DIR / "max_context_results.json"
+    RESULTS_FILE = SCRIPT_DIR / "max_context_results.json"
 HOST = os.getenv("HOST", "0.0.0.0")
 PORT = os.getenv("PORT", "8000")
 

scripts/test_start_vllm.py

@@ -0,0 +1,317 @@
+cat start-vllm
+#!/usr/bin/env python3
+import sys
+import os
+import json
+import shutil
+import tempfile
+import subprocess
+from pathlib import Path
+
+# Add benchmarks dir to path to import config
+# Add benchmarks dir to path to import config
+SCRIPT_DIR = Path(__file__).parent.resolve()
+BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
+OPT_DIR = Path("/opt")
+
+# Check /opt first (Container), then local fallback
+if (OPT_DIR / "run_vllm_bench.py").exists():
+    sys.path.append(str(OPT_DIR))
+else:
+    sys.path.append(str(BENCH_DIR))
+
+try:
+    from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
+except ImportError:
+    print("Error: Could not import run_vllm_bench.py config.")
+    sys.exit(1)
+
+if (OPT_DIR / "max_context_results.json").exists():
+    RESULTS_FILE = OPT_DIR / "max_context_results.json"
+else:
+    RESULTS_FILE = BENCH_DIR / "max_context_results.json"
+HOST = os.getenv("HOST", "0.0.0.0")
+PORT = os.getenv("PORT", "8000")
+
+def check_dependencies():
+    if not shutil.which("dialog"):
+        print("Error: 'dialog' is required. Please install it (apt-get install dialog).")
+        sys.exit(1)
+
+def detect_gpus():
+    """Detects AMD GPUs via rocm-smi or /dev/dri."""
+    try:
+        # Try rocm-smi first
+        res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if res.returncode == 0:
+            count = res.stdout.count("GPU")
+            if count > 0: return count
+    except: pass
+    
+    # Fallback to /dev/dri/render*
+    try:
+        return len(list(Path("/dev/dri").glob("renderD*")))
+    except:
+        return 1
+
+def get_verified_config(model_id, tp_size, max_seqs):
+    """
+    Reads max_context_results.json to find the best verified configuration.
+    Returns dict: {'ctx': int, 'util': float}
+    """
+    default_config = {
+        "ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)),
+        "util": 0.90 # Safe default
+    }
+    
+    if not RESULTS_FILE.exists():
+        return default_config
+
+    try:
+        with open(RESULTS_FILE, "r") as f:
+            data = json.load(f)
+            
+        # Filter for Model + TP + Sequences
+        matches = [r for r in data 
+                  if r["model"] == model_id 
+                  and r["tp"] == tp_size 
+                  and r["max_seqs"] == max_seqs 
+                  and r["status"] == "success"]
+        
+        if not matches:
+            # Fallback 1: Try finding match with SAME TP but ANY Sequences (e.g. 1) to get base context?
+            # Actually, safer to fallback to default or try finding nearest sequence?
+            # Let's try finding exact match first. If fail, return default.
+            return default_config
+            
+        # Sort by Util desc, then Context desc
+        # We prefer higher utilization if available (performance), as long as it is verified success
+        matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True)
+        
+        best = matches[0]
+        return {
+            "ctx": best["max_context_1_user"],
+            "util": float(best["util"])
+        }
+        
+    except Exception as e:
+        return default_config
+
+def run_dialog(args):
+    """Runs dialog and returns stderr (selection)."""
+    with tempfile.NamedTemporaryFile(mode="w+") as tf:
+        cmd = ["dialog"] + args
+        try:
+            subprocess.run(cmd, stderr=tf, check=True)
+            tf.seek(0)
+            return tf.read().strip()
+        except subprocess.CalledProcessError:
+            return None # User cancelled
+
+def nuke_vllm_cache():
+    """Removes vLLM cache directory to fix potential graph/incompatibility issues."""
+    cache = Path.home() / ".cache" / "vllm"
+    if cache.exists():
+        try:
+            print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
+            subprocess.run(["rm", "-rf", str(cache)], check=True)
+            cache.mkdir(parents=True, exist_ok=True)
+            print(" Done.")
+            time.sleep(1)
+        except Exception as e:
+            print(f" Failed: {e}")
+
+def configure_and_launch(model_idx, gpu_count):
+    model_id = MODELS_TO_RUN[model_idx]
+    config = MODEL_TABLE[model_id]
+    
+    # Static Config
+    valid_tps = config.get("valid_tp", [1])
+    max_tp = max(valid_tps) if valid_tps else 1
+    
+    # Defaults
+    current_tp = min(gpu_count, max_tp)
+    current_seqs = 1 # Default to 1 concurrent user/request for stability
+    
+    # Initial Lookup
+    verified = get_verified_config(model_id, current_tp, current_seqs)
+    current_ctx = verified["ctx"]
+    current_util = verified["util"]
+    
+    clear_cache = False
+    use_eager = config.get("enforce_eager", False) # Default to model config, usually False
+    use_rocm_attn = False # Default to Triton
+    
+    name = model_id.split("/")[-1]
+    
+    while True:
+        cache_status = "YES" if clear_cache else "NO"
+        eager_status = "YES" if use_eager else "NO"
+        attn_backend = "ROCm" if use_rocm_attn else "Triton"
+        
+        menu_args = [
+            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+            "--title", f"Configuration: {name}",
+            "--menu", "Customize Launch Parameters:", "22", "65", "9",
+            "1", f"Tensor Parallelism:   {current_tp}",
+            "2", f"Concurrent Requests:  {current_seqs}",
+            "3", f"Context Length:       {current_ctx} (Verified)",
+            "4", f"GPU Utilization:      {current_util} (Verified)",
+            "5", f"Attention Backend:    {attn_backend}",
+            "6", f"Erase vLLM Cache:     {cache_status}",
+            "7", f"Force Eager Mode:     {eager_status}",
+            "8", "LAUNCH SERVER"
+        ]
+        
+        choice = run_dialog(menu_args)
+        if not choice: return False # Back/Cancel
+        
+        if choice == "1":
+            # TP Selection
+            new_tp = run_dialog([
+                "--title", "Tensor Parallelism",
+                "--rangebox", f"Set TP Size (1-{max_tp})", "10", "40", "1", str(max_tp), str(current_tp)
+            ])
+            if new_tp: 
+                new_tp_int = int(new_tp)
+                if new_tp_int != current_tp:
+                    current_tp = new_tp_int
+                    # RE-CALCULATE Config
+                    verified = get_verified_config(model_id, current_tp, current_seqs)
+                    current_ctx = verified["ctx"]
+                    current_util = verified["util"]
+            
+        elif choice == "2":
+            # Max Seqs Selection
+            new_seqs = run_dialog([
+                "--title", "Concurrent Requests",
+                "--menu", "Select Max Concurrent Requests:", "12", "40", "4",
+                "1", "1 (Latency Focus)",
+                "4", "4 (Balanced)",
+                "8", "8 (Throughput)",
+                "16", "16 (Max Load)"
+            ])
+            if new_seqs:
+                current_seqs = int(new_seqs)
+                # RE-CALCULATE Config based on new concurrency
+                verified = get_verified_config(model_id, current_tp, current_seqs)
+                current_ctx = verified["ctx"]
+                current_util = verified["util"]
+
+        elif choice == "3":
+            # Configured Length Override
+            new_ctx = run_dialog([
+                "--title", "Context Length",
+                "--inputbox", f"Override verified limit ({current_ctx}):", "10", "40", str(current_ctx)
+            ])
+            if new_ctx: current_ctx = int(new_ctx)
+
+        elif choice == "4":
+             # Util Override
+             pass 
+
+        elif choice == "5":
+            # Toggle Attention Backend
+            use_rocm_attn = not use_rocm_attn
+
+        elif choice == "6":
+            # Toggle Cache
+            if not clear_cache:
+                # Enabling it -> Show Warning
+                warn_msg = (
+                    "WARNING: Erasing the vLLM cache will remove the compiled compute graphs.\n\n"
+                    "This is useful if you are experiencing crashes, 'invalid graph' errors,\n"
+                    "or have switched vLLM versions recently.\n\n"
+                    "However, the next startup will take longer as graphs are re-compiled.\n\n"
+                    "Are you sure you want to enable this?"
+                )
+                confirm = run_dialog([
+                    "--title", "Erase Cache Warning", 
+                    "--yesno", warn_msg, "12", "60"
+                ])
+                
+                # If confirm is not None (exit 0), it is YES.
+                if confirm is not None:
+                     clear_cache = True
+            else:
+                # Disabling it -> No warning needed
+                clear_cache = False
+             
+        elif choice == "7":
+            # Toggle Eager Mode
+            use_eager = not use_eager
+             
+        elif choice == "8":
+            # Launch
+            break
+            
+    # Build Command
+    subprocess.run(["clear"])
+    
+    if clear_cache:
+        nuke_vllm_cache()
+    
+    cmd = [
+        "vllm", "serve", model_id,
+        "--host", HOST,
+        "--port", PORT,
+        "--tensor-parallel-size", str(current_tp),
+        "--max-num-seqs", str(current_seqs),
+        "--max-model-len", str(current_ctx),
+        "--gpu-memory-utilization", str(current_util),
+        "--dtype", "auto"
+    ]
+    
+    if config.get("trust_remote"): cmd.append("--trust-remote-code")
+    if use_eager: cmd.append("--enforce-eager")
+    
+    # Env Vars
+    env = os.environ.copy()
+    env.update(config.get("env", {}))
+    
+    if use_rocm_attn:
+        env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
+        env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
+        # Optional: Explicitly mention these in print
+        
+    
+    print("\n" + "="*60)
+    print(f" Launching: {name}")
+    print(f" Config:    TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx} | Util={current_util}")
+    print(f" Backend:   {'ROCm' if use_rocm_attn else 'Triton'}")
+    if clear_cache:
+        print(f" Action:    Clearing vLLM Cache (~/.cache/vllm)")
+    print(f" Command:   {' '.join(cmd)}")
+    print("="*60 + "\n")
+    
+    os.execvpe("vllm", cmd, env)
+
+def main():
+    check_dependencies()
+    gpu_count = detect_gpus()
+    
+    while True:
+        # Build Model Menu
+        menu_items = []
+        for i, m_id in enumerate(MODELS_TO_RUN):
+            name = m_id.split("/")[-1]
+            # Pre-calc verified ctx for 'default' TP to show in menu? 
+            # Or just show names. Just names is cleaner.
+            config = MODEL_TABLE[m_id]
+            menu_items.extend([str(i), name])
+            
+        choice = run_dialog([
+            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+            "--title", "Select Model",
+            "--menu", "Choose a model to serve:", "20", "60", "10"
+        ] + menu_items)
+        
+        if not choice:
+            subprocess.run(["clear"])
+            print("Selection cancelled.")
+            sys.exit(0)
+            
+        configure_and_launch(int(choice), gpu_count)
+
+if __name__ == "__main__":
+    main()