import os import subprocess import sys from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings def build_command() -> list[str]: config_file = "config.json" catalog = load_catalog(config_file) runtime = resolve_runtime_settings(catalog) _, updates, env_vars = resolve_model_profile( content=catalog, requested_model=runtime["model_key"], requested_tp=runtime["tensor_parallel_size"], ) for key, value in env_vars.items(): os.environ[key] = value host = str(runtime["openai_host"]) port = str(runtime["openai_port"]) public_model_name = str(runtime["public_model_name"]).strip() api_key = runtime["api_key"] or "" dtype = str(updates["dtype"] or runtime["dtype"]) quantization = str(updates["quantization"] or "").strip() revision = runtime["revision"] or "" cmd = [ sys.executable, "-m", "vllm.entrypoints.openai.api_server", "--host", host, "--port", port, "--model", str(updates["model_name"]), "--served-model-name", public_model_name or str(updates["served_model_name"]), "--tensor-parallel-size", str(updates["tensor_parallel_size"]), "--max-model-len", str(updates["max_model_len"]), "--gpu-memory-utilization", str(updates["gpu_memory_utilization"]), "--max-num-seqs", str(updates["max_num_seqs"]), "--dtype", dtype, ] if updates["trust_remote_code"]: cmd.append("--trust-remote-code") if updates["enforce_eager"]: cmd.append("--enforce-eager") if updates["enable_auto_tool_choice"]: cmd.append("--enable-auto-tool-choice") if updates["tool_call_parser"]: cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])]) if quantization: cmd.extend(["--quantization", quantization]) if revision: cmd.extend(["--revision", revision]) if api_key: cmd.extend(["--api-key", api_key]) return cmd def main() -> None: command = build_command() raise SystemExit(subprocess.call(command)) if __name__ == "__main__": main()