部署的机器
部署的脚本:
#!/bin/bash
# 停止并移除旧容器（如果存在）
docker stop qwen3.5-397b-fp8 2>/dev/null
docker rm qwen3.5-397b-fp8 2>/dev/null
docker run -d \
--gpus all \
--ipc=host \
--shm-size 200g \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--restart unless-stopped \
-p 7380:8000 \
-v /home/rd/models:/models \
--name "qwen3.5-397b-fp8" \
-e VLLM_LOGGING_LEVEL=info \
-e VLLM_LOG_MAX_LEN=999999 \
-e NCCL_IB_DISABLE=0 \
-e NCCL_IB_GID_INDEX=3 \
-e TZ=Asia/Shanghai \
vllm/vllm-openai:latest \
--model /models/Qwen3.5-397B-A17B-FP8 \
--served-model-name Qwen3.5-397B-A17B-FP8 \
--tensor-parallel-size 8 \
--enable-expert-parallel \
--max-model-len 262144 \
--port 8000 \
--host 0.0.0.0 \
--trust-remote-code \
--max-num-seqs 256 \
--gpu-memory-utilization 0.85 \
--enable-auto-tool-choice \
--tool-call-parser qwen3_coder \
--exclude-tools-when-tool-choice-none \
--enable-log-requests
# 查看容器启动日志
echo "容器已启动，开始跟踪日志..."
docker logs -f qwen3.5-397b-fp8测试情况:
curl localhost:7380/v1/models | jq .
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 486 100 486 0 0 138k 0 --:--:-- --:--:-- --:--:-- 158k
{
"object": "list",
"data": [
{
"id": "Qwen3.5-397B-A17B-FP8",
"object": "model",
"created": 1778986981,
"owned_by": "vllm",
"root": "/models/Qwen3.5-397B-A17B-FP8",
"parent": null,
"max_model_len": 262144,
"permission": [
{
"id": "modelperm-8c61f5c1675997a3",
"object": "model_permission",
"created": 1778986981,
"allow_create_engine": false,
"allow_sampling": true,
"allow_logprobs": true,
"allow_search_indices": false,
"allow_view": true,
"allow_fine_tuning": false,
"organization": "*",
"group": null,
"is_blocking": false
}
]
}
]
}运行截图:
