qwenasr-qwen36-voxcmp2 base model. -2026/05/14 night

llm_server/start_vllm.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BUNDLE_DIR="${SCRIPT_DIR}"
+
+IMAGE_TAG="local/vllm-qwen36-nvfp4:v0.19.1-cu130"
+IMAGE_TAR="${BUNDLE_DIR}/image/vllm-qwen36-nvfp4_v0.19.1-cu130.tar"
+MODELSCOPE_DIR="${BUNDLE_DIR}/modelscope"
+CHAT_TEMPLATE="${BUNDLE_DIR}/chat_template.jinja"
+CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-nvfp4}"
+HOST_PORT="${HOST_PORT:-8000}"
+
+if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+  DOCKER=(docker)
+else
+  DOCKER=(sudo docker)
+fi
+
+require_path() {
+  local path="$1"
+  if [[ ! -e "${path}" ]]; then
+    echo "Missing required path: ${path}" >&2
+    exit 1
+  fi
+}
+
+require_path "${IMAGE_TAR}"
+require_path "${MODELSCOPE_DIR}"
+require_path "${CHAT_TEMPLATE}"
+
+mkdir -p "${MODELSCOPE_DIR}/hub/models/Qwen"
+
+MODEL_REAL_DIR="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3___6-35B-A3B-FP8"
+MODEL_LINK="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3.6-35B-A3B-FP8"
+
+require_path "${MODEL_REAL_DIR}"
+
+rm -f "${MODEL_LINK}"
+ln -s "Qwen3___6-35B-A3B-FP8" "${MODEL_LINK}"
+
+if ! "${DOCKER[@]}" image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
+  echo "Loading image ${IMAGE_TAG} from ${IMAGE_TAR}"
+  "${DOCKER[@]}" load -i "${IMAGE_TAR}"
+fi
+
+"${DOCKER[@]}" rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
+
+echo "Starting container ${CONTAINER_NAME} on port ${HOST_PORT}"
+"${DOCKER[@]}" run -d \
+  --name "${CONTAINER_NAME}" \
+  --gpus all \
+  --ipc=host \
+  --network host \
+  -e VLLM_USE_MODELSCOPE=true \
+  -v "${MODELSCOPE_DIR}:/root/.cache/modelscope" \
+  -v "${MODELSCOPE_DIR}:/home/liyh/.cache/modelscope" \
+  -v "${CHAT_TEMPLATE}:/root/chat_template.jinja:ro" \
+  "${IMAGE_TAG}" \
+  Qwen/Qwen3.6-35B-A3B-FP8 \
+  --port "${HOST_PORT}" \
+  --tensor-parallel-size 1 \
+  --max-model-len 131072 \
+  --reasoning-parser qwen3 \
+  --enable-auto-tool-choice \
+  --tool-call-parser qwen3_coder \
+  --api-key semi-ai.tech \
+  --enable-prefix-caching \
+  --max_num_batched_tokens 4096 \
+  --max_num_seqs 8 \
+  --kv-cache-dtype fp8_e4m3 \
+  --chat-template /root/chat_template.jinja \
+  --safetensors-load-strategy eager \
+  --gpu-memory-utilization 0.75\
+  "$@"
+
+echo
+echo "Health check:"
+echo "  curl http://127.0.0.1:${HOST_PORT}/health"
+echo "Logs:"
+echo "  ${DOCKER[*]} logs -f ${CONTAINER_NAME}"