diff --git a/app/start_openai.py b/app/start_openai.py index ba42e73..dd58b21 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -71,12 +71,12 @@ def build_command() -> list[str]: cmd.append("--enable-prefix-caching") if config.get("max_num_batched_tokens", 0) > 0: cmd.extend(["--max-num-batched-tokens", str(config["max_num_batched_tokens"])]) - if updates.get("language_model_only"): + if config.get("language_model_only"): cmd.append("--language-model-only") - speculative_method = str(updates.get("speculative_method") or "").strip() - speculative_model = str(updates.get("speculative_model") or "").strip() - num_speculative_tokens = int(updates.get("num_speculative_tokens") or 0) - speculative_draft_tp = int(updates.get("speculative_draft_tp") or 0) + speculative_method = str(config.get("speculative_method") or "").strip() + speculative_model = str(config.get("speculative_model") or "").strip() + num_speculative_tokens = int(config.get("num_speculative_tokens") or 0) + speculative_draft_tp = int(config.get("speculative_draft_tp") or 0) if speculative_method and speculative_model and num_speculative_tokens > 0: spec_config: dict = { "method": speculative_method,