import json import os import subprocess import sys from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings def build_command() -> list[str]: config_file = "config.json" catalog = load_catalog(config_file) runtime = resolve_runtime_settings(catalog) _, updates, env_vars = resolve_model_profile( content=catalog, requested_model=runtime["model_key"], requested_tp=runtime["tensor_parallel_size"], ) for key, value in env_vars.items(): os.environ[key] = value host = str(runtime["openai_host"]) port = str(runtime["openai_port"]) public_model_name = str(runtime["public_model_name"]).strip() default_enable_thinking = bool(runtime["default_enable_thinking"]) reasoning_enabled = bool(runtime["reasoning_enabled"]) api_key = runtime["api_key"] or "" dtype = str(updates["dtype"] or runtime["dtype"]) quantization = str(updates["quantization"] or "").strip() model_impl = str(updates["model_impl"] or "").strip() reasoning_parser = str(updates["reasoning_parser"] or "").strip() revision = runtime["revision"] or "" cmd = [ sys.executable, "-m", "vllm.entrypoints.openai.api_server", "--host", host, "--port", port, "--model", str(updates["model_name"]), "--served-model-name", public_model_name or str(updates["served_model_name"]), "--tensor-parallel-size", str(updates["tensor_parallel_size"]), "--max-model-len", str(updates["max_model_len"]), "--gpu-memory-utilization", str(updates["gpu_memory_utilization"]), "--max-num-seqs", str(updates["max_num_seqs"]), "--dtype", dtype, ] if updates["trust_remote_code"]: cmd.append("--trust-remote-code") if updates["enforce_eager"]: cmd.append("--enforce-eager") if updates["enable_auto_tool_choice"]: cmd.append("--enable-auto-tool-choice") if updates["tool_call_parser"]: cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])]) cmd.extend( [ "--default-chat-template-kwargs", json.dumps({"enable_thinking": default_enable_thinking}), ] ) if reasoning_enabled and reasoning_parser: cmd.extend(["--reasoning-parser", reasoning_parser]) if quantization: cmd.extend(["--quantization", quantization]) if model_impl: cmd.extend(["--model-impl", model_impl]) if revision: cmd.extend(["--revision", revision]) if api_key: cmd.extend(["--api-key", api_key]) if updates.get("kv_cache_dtype"): cmd.extend(["--kv-cache-dtype", str(updates["kv_cache_dtype"])]) if updates.get("enable_prefix_caching"): cmd.append("--enable-prefix-caching") if updates.get("max_num_batched_tokens", 0) > 0: cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])]) if updates.get("language_model_only"): cmd.append("--language-model-only") return cmd def main() -> None: command = build_command() raise SystemExit(subprocess.call(command)) if __name__ == "__main__": main()