2026-03-29 05:09:37 +08:00
|
|
|
MODEL_CONFIG_FILE=config.json
|
|
|
|
|
MODEL_KEY=Qwen3.5-35B-A3B-GPTQ-Int4
|
|
|
|
|
MODEL_NAME=
|
2026-03-29 05:06:19 +08:00
|
|
|
HOST=0.0.0.0
|
|
|
|
|
PORT=8000
|
|
|
|
|
TENSOR_PARALLEL_SIZE=2
|
|
|
|
|
GPU_MEMORY_UTILIZATION=0.92
|
|
|
|
|
MAX_MODEL_LEN=8192
|
|
|
|
|
MAX_NUM_SEQS=64
|
2026-03-29 05:09:37 +08:00
|
|
|
MAX_TOKENS=4096
|
2026-03-29 05:06:19 +08:00
|
|
|
DTYPE=bfloat16
|
|
|
|
|
ENFORCE_EAGER=false
|
|
|
|
|
TRUST_REMOTE_CODE=false
|
2026-03-29 05:09:37 +08:00
|
|
|
TOOL_CALL_PARSER=
|
|
|
|
|
ENABLE_AUTO_TOOL_CHOICE=false
|
2026-03-29 05:06:19 +08:00
|
|
|
REVISION=
|
|
|
|
|
API_KEY=
|
