x

app/model_catalog.py

@@ -104,6 +104,19 @@ def resolve_model_profile(
     resolved_tp = requested_tp
     if valid_tp and resolved_tp not in valid_tp:
         resolved_tp = valid_tp[0]
+    speculative = dict(profile.get("speculative", {}))
+    speculative_method = _to_str(speculative.get("method"))
+    speculative_model_path = _to_str(speculative.get("model"))
+    num_speculative_tokens = _to_int(speculative.get("num_speculative_tokens"), 0)
+    speculative_draft_tp = _to_int(speculative.get("draft_tensor_parallel_size"), 0)
+    if speculative_method and speculative_model_path:
+        resolved_speculative_model = speculative_model_path
+        if not speculative_model_path.startswith("/"):
+            if not model_root:
+                raise ValueError("config.json model_root cannot be empty when speculative model path is relative")
+            resolved_speculative_model = _join_posix(model_root, speculative_model_path)
+    else:
+        resolved_speculative_model = ""
     updates = {
         "selected_model": model_key,
         "model_name": _resolve_profile_model_path(profile, model_root, model_key),
@@ -125,6 +138,10 @@ def resolve_model_profile(
         "enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False),
         "max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0),
         "language_model_only": _to_bool(profile.get("language_model_only"), False),
+        "speculative_method": speculative_method,
+        "speculative_model": resolved_speculative_model,
+        "num_speculative_tokens": num_speculative_tokens,
+        "speculative_draft_tp": speculative_draft_tp,
     }
     env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
     env_vars["HF_HUB_OFFLINE"] = "1"