diff --git a/README.md b/README.md index c8160eb..2518a82 100644 --- a/README.md +++ b/README.md @@ -104,5 +104,6 @@ curl -X POST "http://localhost:8001/v1/chat/completions" \ ## 常见故障排查 - 报错 `model type ... Transformers does not recognize this architecture` 时,说明当前模型与镜像内依赖不兼容,建议更换模型或升级镜像版本。 +- 报错 `model config (gptq) does not match quantization argument (gptq_marlin)` 时,将该模型配置改为 `dtype=float16` 且 `quantization=gptq`。 - 若模型目录存在但仍加载失败,检查挂载路径是否为 `/opt/model:/opt/model:ro`,并确认容器内可见模型文件。 - 如果看到 `No services to build`,说明未触发重建;需要先执行 `docker compose build --no-cache` 再 `up`。 diff --git a/config.json b/config.json index c0462c0..c2c1c53 100644 --- a/config.json +++ b/config.json @@ -65,7 +65,7 @@ "Qwen3.5-35B-A3B-GPTQ-Int4": { "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4", "dtype": "float16", - "quantization": "gptq_marlin", + "quantization": "gptq", "ctx": "65536", "trust_remote": true, "valid_tp": [1, 2], @@ -76,6 +76,20 @@ "enable_auto_tool_choice": true, "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4", "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4" + }, + "Qwen3.5-35B-A3B-FP8": { + "local_path": "Qwen3.5-35B-A3B-FP8", + "dtype": "bfloat16", + "ctx": "65536", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + "gpu_util": "0.98", + "tool_call_parser": "qwen3_xml", + "enable_auto_tool_choice": true, + "served_model_name": "Qwen3.5-35B-A3B-FP8", + "hf_model_id": "Qwen/Qwen3.5-35B-A3B-FP8" } } }