cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
SZCJW
2026-03-28 13:54:48 +08:00
parent 7ea8cf48c3
commit b563d85519
2 changed files with 64 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.json
+54
View File
@@ -1,28 +1,78 @@
{
"default_model": "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"models": {
"Meta-Llama-3.1-8B-Instruct": {
"ctx": "65536",
"trust_remote": false,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "meta-llama/Meta-Llama-3.1-8B-Instruct"
},
"gpt-oss-20b": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "8192",
"gpu_util": "0.98",
"hf_model_id": "openai/gpt-oss-20b"
},
"Qwen3-14B-FP8-dynamic": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "RedHatAI/Qwen3-14B-FP8-dynamic"
},
"Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
"ctx": "24576",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit"
},
"Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
"ctx": "20480",
"trust_remote": true,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "16384",
"gpu_util": "0.98",
"enforce_eager": false,
"env": {"VLLM_USE_TRITON_AWQ": "1"},
"hf_model_id": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit"
},
"gemma-3-27b-it-FP8-dynamic": {
"ctx": "29000",
"trust_remote": true,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "29000",
"gpu_util": "0.94",
"hf_model_id": "RedHatAI/gemma-3-27b-it-FP8-dynamic"
},
"gemma-3-12b-it-FP8-dynamic": {
"ctx": "9900",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "9900",
"gpu_util": "0.98",
"hf_model_id": "RedHatAI/gemma-3-12b-it-FP8-dynamic"
},
"DeepSeek-R1-Distill-Qwen-14B": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"DeepSeek-R1-Distill-Qwen-32B-AWQ": {
@@ -31,6 +81,7 @@
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B-AWQ"
},
"GLM-4.7-Flash-AWQ": {
@@ -39,6 +90,7 @@
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "THUDM/GLM-4.7-Flash-AWQ"
},
"Qwen3.5-35B-A3B-GPTQ-Int4": {
@@ -47,6 +99,7 @@
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
},
"Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit": {
@@ -55,6 +108,7 @@
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"gpu_util": "0.98",
"hf_model_id": "Qwen/Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit"
}
}
scripts/start_vllm.py
+10 -1
View File
@@ -114,7 +114,16 @@ def launch_model(model_id, config, model_path, gpu_count):
# Get configuration
valid_tp = config.get("valid_tp", [1])
max_tp = max(valid_tp) if valid_tp else 1
tp_size = min(gpu_count, max_tp)
# Check for manual TP_SIZE override
tp_size_env = os.getenv("TP_SIZE")
if tp_size_env:
tp_size = int(tp_size_env)
log(f"TP_SIZE environment variable set: {tp_size}")
if tp_size not in valid_tp:
log(f"WARNING: TP_SIZE={tp_size} is not in valid_tp={valid_tp}, proceeding anyway")
else:
tp_size = min(gpu_count, max_tp)
ctx = int(config.get("ctx", 8192))
max_seqs = int(config.get("max_num_seqs", 64))