cjw/rocm_vllm_nightly
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
chenjw28
2026-04-23 15:08:27 +08:00
parent f595b53d70
commit b534ceac58
3 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/model_catalog.py
+1
View File
@@ -124,6 +124,7 @@ def resolve_model_profile(
"kv_cache_dtype": _to_str(profile.get("kv_cache_dtype")), "kv_cache_dtype": _to_str(profile.get("kv_cache_dtype")),
"enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False), "enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False),
"max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0), "max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0),
"language_model_only": _to_bool(profile.get("language_model_only"), False),
} }
env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()} env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
env_vars["HF_HUB_OFFLINE"] = "1" env_vars["HF_HUB_OFFLINE"] = "1"
app/start_openai.py
+2
View File
@@ -81,6 +81,8 @@ def build_command() -> list[str]:
cmd.append("--enable-prefix-caching") cmd.append("--enable-prefix-caching")
if updates.get("max_num_batched_tokens", 0) > 0: if updates.get("max_num_batched_tokens", 0) > 0:
cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])]) cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])])
if updates.get("language_model_only"):
cmd.append("--language-model-only")
return cmd return cmd
config.json
+4 -2
View File
@@ -38,12 +38,13 @@
"tool_call_parser": "qwen3_coder", "tool_call_parser": "qwen3_coder",
"reasoning_parser": "qwen3", "reasoning_parser": "qwen3",
"enable_auto_tool_choice": true, "enable_auto_tool_choice": true,
"language_model_only": true,
"served_model_name": "Qwen3.6-35B-A3B-FP8", "served_model_name": "Qwen3.6-35B-A3B-FP8",
"hf_model_id": "Qwen/Qwen3.6-35B-A3B-FP8" "hf_model_id": "Qwen/Qwen3.6-35B-A3B-FP8"
}, },
"Qwen3.6-27B": { "Qwen3.6-27B": {
"local_path": "Qwen3.6-27B", "local_path": "Qwen3.6-27B",
"ctx": "262144", "ctx": "131072",
"max_tokens": "32768", "max_tokens": "32768",
"max_num_batched_tokens": 16384, "max_num_batched_tokens": 16384,
"trust_remote": true, "trust_remote": true,
@@ -53,12 +54,13 @@
2 2
], ],
"max_num_seqs": 32, "max_num_seqs": 32,
"gpu_util": "0.90", "gpu_util": "0.92",
"kv_cache_dtype": "fp8", "kv_cache_dtype": "fp8",
"enable_prefix_caching": true, "enable_prefix_caching": true,
"tool_call_parser": "qwen3_coder", "tool_call_parser": "qwen3_coder",
"reasoning_parser": "qwen3", "reasoning_parser": "qwen3",
"enable_auto_tool_choice": true, "enable_auto_tool_choice": true,
"language_model_only": true,
"served_model_name": "Qwen3.6-27B", "served_model_name": "Qwen3.6-27B",
"hf_model_id": "Qwen/Qwen3.6-27B" "hf_model_id": "Qwen/Qwen3.6-27B"
}, },