73 lines
2.2 KiB
Python
73 lines
2.2 KiB
Python
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings
|
|
|
|
|
|
def build_command() -> list[str]:
|
|
config_file = "config.json"
|
|
catalog = load_catalog(config_file)
|
|
runtime = resolve_runtime_settings(catalog)
|
|
_, updates, env_vars = resolve_model_profile(
|
|
content=catalog,
|
|
requested_model=runtime["model_key"],
|
|
requested_tp=runtime["tensor_parallel_size"],
|
|
)
|
|
for key, value in env_vars.items():
|
|
os.environ[key] = value
|
|
host = str(runtime["openai_host"])
|
|
port = str(runtime["openai_port"])
|
|
public_model_name = str(runtime["public_model_name"]).strip()
|
|
api_key = runtime["api_key"] or ""
|
|
dtype = str(updates["dtype"] or runtime["dtype"])
|
|
quantization = str(updates["quantization"] or "").strip()
|
|
revision = runtime["revision"] or ""
|
|
cmd = [
|
|
sys.executable,
|
|
"-m",
|
|
"vllm.entrypoints.openai.api_server",
|
|
"--host",
|
|
host,
|
|
"--port",
|
|
port,
|
|
"--model",
|
|
str(updates["model_name"]),
|
|
"--served-model-name",
|
|
public_model_name or str(updates["served_model_name"]),
|
|
"--tensor-parallel-size",
|
|
str(updates["tensor_parallel_size"]),
|
|
"--max-model-len",
|
|
str(updates["max_model_len"]),
|
|
"--gpu-memory-utilization",
|
|
str(updates["gpu_memory_utilization"]),
|
|
"--max-num-seqs",
|
|
str(updates["max_num_seqs"]),
|
|
"--dtype",
|
|
dtype,
|
|
]
|
|
if updates["trust_remote_code"]:
|
|
cmd.append("--trust-remote-code")
|
|
if updates["enforce_eager"]:
|
|
cmd.append("--enforce-eager")
|
|
if updates["enable_auto_tool_choice"]:
|
|
cmd.append("--enable-auto-tool-choice")
|
|
if updates["tool_call_parser"]:
|
|
cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
|
|
if quantization:
|
|
cmd.extend(["--quantization", quantization])
|
|
if revision:
|
|
cmd.extend(["--revision", revision])
|
|
if api_key:
|
|
cmd.extend(["--api-key", api_key])
|
|
return cmd
|
|
|
|
|
|
def main() -> None:
|
|
command = build_command()
|
|
raise SystemExit(subprocess.call(command))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|