82 lines
2.2 KiB
Bash
82 lines
2.2 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
BUNDLE_DIR="${SCRIPT_DIR}"
|
|
|
|
IMAGE_TAG="local/vllm-qwen36-nvfp4:v0.19.1-cu130"
|
|
IMAGE_TAR="${BUNDLE_DIR}/image/vllm-qwen36-nvfp4_v0.19.1-cu130.tar"
|
|
MODELSCOPE_DIR="${BUNDLE_DIR}/modelscope"
|
|
CHAT_TEMPLATE="${BUNDLE_DIR}/chat_template.jinja"
|
|
CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-nvfp4}"
|
|
HOST_PORT="${HOST_PORT:-8000}"
|
|
|
|
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
|
|
DOCKER=(docker)
|
|
else
|
|
DOCKER=(sudo docker)
|
|
fi
|
|
|
|
require_path() {
|
|
local path="$1"
|
|
if [[ ! -e "${path}" ]]; then
|
|
echo "Missing required path: ${path}" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
require_path "${IMAGE_TAR}"
|
|
require_path "${MODELSCOPE_DIR}"
|
|
require_path "${CHAT_TEMPLATE}"
|
|
|
|
mkdir -p "${MODELSCOPE_DIR}/hub/models/Qwen"
|
|
|
|
MODEL_REAL_DIR="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3___6-35B-A3B-FP8"
|
|
MODEL_LINK="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3.6-35B-A3B-FP8"
|
|
|
|
require_path "${MODEL_REAL_DIR}"
|
|
|
|
rm -f "${MODEL_LINK}"
|
|
ln -s "Qwen3___6-35B-A3B-FP8" "${MODEL_LINK}"
|
|
|
|
if ! "${DOCKER[@]}" image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
|
|
echo "Loading image ${IMAGE_TAG} from ${IMAGE_TAR}"
|
|
"${DOCKER[@]}" load -i "${IMAGE_TAR}"
|
|
fi
|
|
|
|
"${DOCKER[@]}" rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
|
|
|
|
echo "Starting container ${CONTAINER_NAME} on port ${HOST_PORT}"
|
|
"${DOCKER[@]}" run -d \
|
|
--name "${CONTAINER_NAME}" \
|
|
--gpus all \
|
|
--ipc=host \
|
|
--network host \
|
|
-e VLLM_USE_MODELSCOPE=true \
|
|
-v "${MODELSCOPE_DIR}:/root/.cache/modelscope" \
|
|
-v "${MODELSCOPE_DIR}:/home/liyh/.cache/modelscope" \
|
|
-v "${CHAT_TEMPLATE}:/root/chat_template.jinja:ro" \
|
|
"${IMAGE_TAG}" \
|
|
Qwen/Qwen3.6-35B-A3B-FP8 \
|
|
--port "${HOST_PORT}" \
|
|
--tensor-parallel-size 1 \
|
|
--max-model-len 131072 \
|
|
--reasoning-parser qwen3 \
|
|
--enable-auto-tool-choice \
|
|
--tool-call-parser qwen3_coder \
|
|
--api-key semi-ai.tech \
|
|
--enable-prefix-caching \
|
|
--max_num_batched_tokens 4096 \
|
|
--max_num_seqs 8 \
|
|
--kv-cache-dtype fp8_e4m3 \
|
|
--chat-template /root/chat_template.jinja \
|
|
--safetensors-load-strategy eager \
|
|
--gpu-memory-utilization 0.75\
|
|
"$@"
|
|
|
|
echo
|
|
echo "Health check:"
|
|
echo " curl http://127.0.0.1:${HOST_PORT}/health"
|
|
echo "Logs:"
|
|
echo " ${DOCKER[*]} logs -f ${CONTAINER_NAME}"
|