x

app/model_catalog.py

@@ -104,6 +104,19 @@ def resolve_model_profile(
     resolved_tp = requested_tp
     if valid_tp and resolved_tp not in valid_tp:
         resolved_tp = valid_tp[0]
+    speculative = dict(profile.get("speculative", {}))
+    speculative_method = _to_str(speculative.get("method"))
+    speculative_model_path = _to_str(speculative.get("model"))
+    num_speculative_tokens = _to_int(speculative.get("num_speculative_tokens"), 0)
+    speculative_draft_tp = _to_int(speculative.get("draft_tensor_parallel_size"), 0)
+    if speculative_method and speculative_model_path:
+        resolved_speculative_model = speculative_model_path
+        if not speculative_model_path.startswith("/"):
+            if not model_root:
+                raise ValueError("config.json model_root cannot be empty when speculative model path is relative")
+            resolved_speculative_model = _join_posix(model_root, speculative_model_path)
+    else:
+        resolved_speculative_model = ""
     updates = {
         "selected_model": model_key,
         "model_name": _resolve_profile_model_path(profile, model_root, model_key),
@@ -125,6 +138,10 @@ def resolve_model_profile(
         "enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False),
         "max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0),
         "language_model_only": _to_bool(profile.get("language_model_only"), False),
+        "speculative_method": speculative_method,
+        "speculative_model": resolved_speculative_model,
+        "num_speculative_tokens": num_speculative_tokens,
+        "speculative_draft_tp": speculative_draft_tp,
     }
     env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
     env_vars["HF_HUB_OFFLINE"] = "1"

app/start_openai.py

@@ -83,6 +83,19 @@ def build_command() -> list[str]:
         cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])])
     if updates.get("language_model_only"):
         cmd.append("--language-model-only")
+    speculative_method = str(updates.get("speculative_method") or "").strip()
+    speculative_model = str(updates.get("speculative_model") or "").strip()
+    num_speculative_tokens = int(updates.get("num_speculative_tokens") or 0)
+    speculative_draft_tp = int(updates.get("speculative_draft_tp") or 0)
+    if speculative_method and speculative_model and num_speculative_tokens > 0:
+        spec_config: dict = {
+            "method": speculative_method,
+            "model": speculative_model,
+            "num_speculative_tokens": num_speculative_tokens,
+        }
+        if speculative_draft_tp > 0:
+            spec_config["draft_tensor_parallel_size"] = speculative_draft_tp
+        cmd.extend(["--speculative-config", json.dumps(spec_config)])
     return cmd
 
 

config.json

@@ -20,7 +20,7 @@
   "revision": "",
   "models": {
     "default": "Qwen3.5-35B-A3B-GPTQ-Int4",
-    "selected": "Qwen3.6-35B-A3B-FP8",
+    "selected": "Qwen3.6-27B-FP8",
     "profiles": {
       "Qwen3.6-35B-A3B-FP8": {
         "local_path": "Qwen3.6-35B-A3B-FP8",
@@ -87,7 +87,13 @@
         "enable_auto_tool_choice": true,
         "language_model_only": true,
         "served_model_name": "Qwen3.6-27B-FP8",
-        "hf_model_id": "Qwen/Qwen3.6-27B-FP8"
+        "hf_model_id": "Qwen/Qwen3.6-27B-FP8",
+        "speculative": {
+          "method": "dflash",
+          "model": "Qwen3.6-27B-DFlash",
+          "num_speculative_tokens": 8,
+          "draft_tensor_parallel_size": 1
+        }
       },
       "Qwen3.5-35B-A3B-FP8": {
         "local_path": "Qwen3.5-35B-A3B-FP8",