cjw/rocm_vllm_nightly
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
chenjw28
2026-05-20 10:39:32 +08:00
parent f3e90d2f17
commit 18ba470f4c
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/start_openai.py
+5 -5
View File
@@ -71,12 +71,12 @@ def build_command() -> list[str]:
cmd.append("--enable-prefix-caching") cmd.append("--enable-prefix-caching")
if config.get("max_num_batched_tokens", 0) > 0: if config.get("max_num_batched_tokens", 0) > 0:
cmd.extend(["--max-num-batched-tokens", str(config["max_num_batched_tokens"])]) cmd.extend(["--max-num-batched-tokens", str(config["max_num_batched_tokens"])])
if updates.get("language_model_only"): if config.get("language_model_only"):
cmd.append("--language-model-only") cmd.append("--language-model-only")
speculative_method = str(updates.get("speculative_method") or "").strip() speculative_method = str(config.get("speculative_method") or "").strip()
speculative_model = str(updates.get("speculative_model") or "").strip() speculative_model = str(config.get("speculative_model") or "").strip()
num_speculative_tokens = int(updates.get("num_speculative_tokens") or 0) num_speculative_tokens = int(config.get("num_speculative_tokens") or 0)
speculative_draft_tp = int(updates.get("speculative_draft_tp") or 0) speculative_draft_tp = int(config.get("speculative_draft_tp") or 0)
if speculative_method and speculative_model and num_speculative_tokens > 0: if speculative_method and speculative_model and num_speculative_tokens > 0:
spec_config: dict = { spec_config: dict = {
"method": speculative_method, "method": speculative_method,