x

2026-03-29 05:22:25 +08:00
parent 6790945334
commit 4db5de796a
5 changed files with 129 additions and 2 deletions
@@ -0,0 +1,68 @@
+import os
+import subprocess
+import sys
+
+from app.model_catalog import resolve_model_profile
+
+
+def build_command() -> list[str]:
+    config_file = os.getenv("MODEL_CONFIG_FILE", "config.json")
+    model_key = os.getenv("MODEL_KEY")
+    requested_tp = int(os.getenv("TENSOR_PARALLEL_SIZE", "2"))
+    _, updates, env_vars = resolve_model_profile(
+        catalog_path=config_file,
+        requested_model=model_key,
+        requested_tp=requested_tp,
+    )
+    for key, value in env_vars.items():
+        os.environ[key] = value
+    host = os.getenv("OPENAI_HOST", "0.0.0.0")
+    port = os.getenv("OPENAI_PORT", "8001")
+    dtype = os.getenv("DTYPE", "bfloat16")
+    revision = os.getenv("REVISION", "").strip()
+    api_key = os.getenv("API_KEY", "").strip()
+    cmd = [
+        sys.executable,
+        "-m",
+        "vllm.entrypoints.openai.api_server",
+        "--host",
+        host,
+        "--port",
+        port,
+        "--model",
+        str(updates["model_name"]),
+        "--served-model-name",
+        str(updates["served_model_name"]),
+        "--tensor-parallel-size",
+        str(updates["tensor_parallel_size"]),
+        "--max-model-len",
+        str(updates["max_model_len"]),
+        "--gpu-memory-utilization",
+        str(updates["gpu_memory_utilization"]),
+        "--max-num-seqs",
+        str(updates["max_num_seqs"]),
+        "--dtype",
+        dtype,
+    ]
+    if updates["trust_remote_code"]:
+        cmd.append("--trust-remote-code")
+    if updates["enforce_eager"]:
+        cmd.append("--enforce-eager")
+    if updates["enable_auto_tool_choice"]:
+        cmd.append("--enable-auto-tool-choice")
+    if updates["tool_call_parser"]:
+        cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
+    if revision:
+        cmd.extend(["--revision", revision])
+    if api_key:
+        cmd.extend(["--api-key", api_key])
+    return cmd
+
+
+def main() -> None:
+    command = build_command()
+    raise SystemExit(subprocess.call(command))
+
+
+if __name__ == "__main__":
+    main()