x

config.json

@@ -1,28 +1,78 @@
 {
   "default_model": "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
   "models": {
+    "Meta-Llama-3.1-8B-Instruct": {
+      "ctx": "65536",
+      "trust_remote": false,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "hf_model_id": "meta-llama/Meta-Llama-3.1-8B-Instruct"
+    },
     "gpt-oss-20b": {
       "ctx": "32768",
       "trust_remote": true,
       "valid_tp": [1, 2],
       "max_num_seqs": "64",
       "max_tokens": "8192",
+      "gpu_util": "0.98",
       "hf_model_id": "openai/gpt-oss-20b"
     },
+    "Qwen3-14B-FP8-dynamic": {
+      "ctx": "32768",
+      "trust_remote": true,
+      "valid_tp": [1],
+      "max_num_seqs": "64",
+      "max_tokens": "32768",
+      "gpu_util": "0.98",
+      "hf_model_id": "RedHatAI/Qwen3-14B-FP8-dynamic"
+    },
     "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
       "ctx": "24576",
       "trust_remote": true,
       "valid_tp": [1, 2],
       "max_num_seqs": "64",
       "max_tokens": "32768",
+      "gpu_util": "0.98",
       "hf_model_id": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit"
     },
+    "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
+      "ctx": "20480",
+      "trust_remote": true,
+      "valid_tp": [2],
+      "max_num_seqs": "32",
+      "max_tokens": "16384",
+      "gpu_util": "0.98",
+      "enforce_eager": false,
+      "env": {"VLLM_USE_TRITON_AWQ": "1"},
+      "hf_model_id": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit"
+    },
+    "gemma-3-27b-it-FP8-dynamic": {
+      "ctx": "29000",
+      "trust_remote": true,
+      "valid_tp": [2],
+      "max_num_seqs": "32",
+      "max_tokens": "29000",
+      "gpu_util": "0.94",
+      "hf_model_id": "RedHatAI/gemma-3-27b-it-FP8-dynamic"
+    },
+    "gemma-3-12b-it-FP8-dynamic": {
+      "ctx": "9900",
+      "trust_remote": true,
+      "valid_tp": [1, 2],
+      "max_num_seqs": "64",
+      "max_tokens": "9900",
+      "gpu_util": "0.98",
+      "hf_model_id": "RedHatAI/gemma-3-12b-it-FP8-dynamic"
+    },
     "DeepSeek-R1-Distill-Qwen-14B": {
       "ctx": "32768",
       "trust_remote": true,
       "valid_tp": [1],
       "max_num_seqs": "64",
       "max_tokens": "32768",
+      "gpu_util": "0.98",
       "hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
     },
     "DeepSeek-R1-Distill-Qwen-32B-AWQ": {
@@ -31,6 +81,7 @@
       "valid_tp": [1, 2],
       "max_num_seqs": "64",
       "max_tokens": "32768",
+      "gpu_util": "0.98",
       "hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B-AWQ"
     },
     "GLM-4.7-Flash-AWQ": {
@@ -39,6 +90,7 @@
       "valid_tp": [1, 2],
       "max_num_seqs": "64",
       "max_tokens": "32768",
+      "gpu_util": "0.98",
       "hf_model_id": "THUDM/GLM-4.7-Flash-AWQ"
     },
     "Qwen3.5-35B-A3B-GPTQ-Int4": {
@@ -47,6 +99,7 @@
       "valid_tp": [1, 2],
       "max_num_seqs": "64",
       "max_tokens": "32768",
+      "gpu_util": "0.98",
       "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
     },
     "Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit": {
@@ -55,6 +108,7 @@
       "valid_tp": [1, 2],
       "max_num_seqs": "64",
       "max_tokens": "32768",
+      "gpu_util": "0.98",
       "hf_model_id": "Qwen/Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit"
     }
   }

scripts/start_vllm.py

@@ -114,7 +114,16 @@ def launch_model(model_id, config, model_path, gpu_count):
     # Get configuration
     valid_tp = config.get("valid_tp", [1])
     max_tp = max(valid_tp) if valid_tp else 1
-    tp_size = min(gpu_count, max_tp)
+    
+    # Check for manual TP_SIZE override
+    tp_size_env = os.getenv("TP_SIZE")
+    if tp_size_env:
+        tp_size = int(tp_size_env)
+        log(f"TP_SIZE environment variable set: {tp_size}")
+        if tp_size not in valid_tp:
+            log(f"WARNING: TP_SIZE={tp_size} is not in valid_tp={valid_tp}, proceeding anyway")
+    else:
+        tp_size = min(gpu_count, max_tp)
     
     ctx = int(config.get("ctx", 8192))
     max_seqs = int(config.get("max_num_seqs", 64))