diff --git a/README.md b/README.md index 2518a82..2f482dc 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ 项目只读取一个配置文件:`config.json`。 - `services.openai.host` / `services.openai.port`:OpenAI 协议服务监听地址与端口(默认 `0.0.0.0:8001`) +- `public_model_name`:对外固定模型名,切换底层模型时可保持调用方参数不变 - `api_key`:OpenAI 接口访问密钥 - `tensor_parallel_size`:张量并行数,双卡建议 `2` - `dtype`:推理精度,默认 `bfloat16` @@ -82,14 +83,14 @@ curl http://localhost:/v1/models curl -X POST "http://localhost:8001/v1/chat/completions" \ -H "Content-Type: application/json" \ -H "Authorization: Bearer " \ - -d "{\"model\":\"Qwen3.5-35B-A3B-GPTQ-Int4\",\"messages\":[{\"role\":\"user\",\"content\":\"你好,介绍一下你自己\"}],\"temperature\":0.7}" + -d "{\"model\":\"Qwen_local_model\",\"messages\":[{\"role\":\"user\",\"content\":\"你好,介绍一下你自己\"}],\"temperature\":0.7}" ``` ## OpenClaw 调用说明 - Base URL 使用 `http://<服务器IP>:8001/v1` - API Key 使用 `config.json` 中 `api_key` -- 模型名使用 `config.json` 中 `models.profiles.<模型名>.served_model_name` +- 模型名固定使用 `config.json` 中 `public_model_name`(默认 `Qwen_local_model`) - 若使用工具调用,`config.json` 中应配置 `tool_call_parser` 与 `enable_auto_tool_choice` - 服务强制离线模式,不会回退到 Hugging Face 远程下载 - 所有路径按 Ubuntu 规范填写,本地模型建议使用 `/opt/model/<模型目录>` @@ -105,5 +106,6 @@ curl -X POST "http://localhost:8001/v1/chat/completions" \ - 报错 `model type ... Transformers does not recognize this architecture` 时,说明当前模型与镜像内依赖不兼容,建议更换模型或升级镜像版本。 - 报错 `model config (gptq) does not match quantization argument (gptq_marlin)` 时,将该模型配置改为 `dtype=float16` 且 `quantization=gptq`。 +- 报错 `RPC call to sample_tokens timed out` 或出现 `GPU core dump` 时,先下调模型配置为更稳参数:`ctx=32768`、`max_num_seqs=4`、`max_tokens=2048`、`gpu_util=0.90`,并开启 `enforce_eager=true`。 - 若模型目录存在但仍加载失败,检查挂载路径是否为 `/opt/model:/opt/model:ro`,并确认容器内可见模型文件。 - 如果看到 `No services to build`,说明未触发重建;需要先执行 `docker compose build --no-cache` 再 `up`。 diff --git a/app/config.py b/app/config.py index 821d65e..0d98a35 100644 --- a/app/config.py +++ b/app/config.py @@ -17,6 +17,7 @@ class Settings(BaseModel): port: int = 8000 openai_host: str = "0.0.0.0" openai_port: int = 8001 + public_model_name: str = "Qwen_local_model" model_root: str = "/opt/model" offline_mode: bool = True max_model_len: int = 8192 @@ -44,6 +45,7 @@ def get_settings() -> Settings: port=runtime["port"], openai_host=runtime["openai_host"], openai_port=runtime["openai_port"], + public_model_name=runtime["public_model_name"], model_root=runtime["model_root"], offline_mode=runtime["offline_mode"], api_key=runtime["api_key"], diff --git a/app/model_catalog.py b/app/model_catalog.py index 75fc3e8..3a3c3ba 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -68,6 +68,7 @@ def resolve_runtime_settings(content: dict[str, Any]) -> dict[str, Any]: "port": _to_int(api_service.get("port"), 8000), "openai_host": str(openai_service.get("host", "0.0.0.0")), "openai_port": _to_int(openai_service.get("port"), 8001), + "public_model_name": _to_str(content.get("public_model_name"), "Qwen_local_model"), "api_key": str(content.get("api_key", "")).strip() or None, "tensor_parallel_size": _to_int(content.get("tensor_parallel_size"), 2), "dtype": str(content.get("dtype", "bfloat16")), diff --git a/app/start_openai.py b/app/start_openai.py index 6df4540..b32479f 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -18,6 +18,7 @@ def build_command() -> list[str]: os.environ[key] = value host = str(runtime["openai_host"]) port = str(runtime["openai_port"]) + public_model_name = str(runtime["public_model_name"]).strip() api_key = runtime["api_key"] or "" dtype = str(updates["dtype"] or runtime["dtype"]) quantization = str(updates["quantization"] or "").strip() @@ -33,7 +34,7 @@ def build_command() -> list[str]: "--model", str(updates["model_name"]), "--served-model-name", - str(updates["served_model_name"]), + public_model_name or str(updates["served_model_name"]), "--tensor-parallel-size", str(updates["tensor_parallel_size"]), "--max-model-len", diff --git a/config.json b/config.json index d682453..fac72a3 100644 --- a/config.json +++ b/config.json @@ -9,6 +9,7 @@ "port": 8001 } }, + "public_model_name": "Qwen_local_model", "api_key": "sk-szcjw", "tensor_parallel_size": 2, "dtype": "bfloat16", @@ -70,9 +71,10 @@ "ctx": "65536", "trust_remote": true, "valid_tp": [1, 2], - "max_num_seqs": "8", - "max_tokens": "8192", - "gpu_util": "0.92", + "max_num_seqs": "4", + "max_tokens": "4096", + "gpu_util": "0.94", + "enforce_eager": true, "tool_call_parser": "qwen3_xml", "enable_auto_tool_choice": true, "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4",