qwenasr-qwen36-voxcmp2 base model. -2026/05/14 night

This commit is contained in:
minorQAQ committed 2026-05-15 16:05:32 +08:00
commit 110fe78016
80 files changed
+6057

No files matched your search

+35
View File
@@ -0,0 +1,35 @@
This bundle is self-contained for running `Qwen/Qwen3.6-35B-A3B-FP8` with
`local/vllm-qwen36-nvfp4:v0.19.1-cu130` on another Spark machine.
Contents:
- `image/`: Docker image tarball
- `modelscope/`: bundled ModelScope cache for the model weights
- `chat_template.jinja`: chat template mounted into the container
- `start_vllm.sh`: loads the image if needed and starts the container
- `stop_vllm.sh`: stops the container
Usage after extraction:
```bash
cd vllm-qwen36-bundle
chmod +x start_vllm.sh stop_vllm.sh
./start_vllm.sh
```
Optional overrides:
- `HOST_PORT=8001 ./start_vllm.sh`
- `CONTAINER_NAME=my-vllm ./start_vllm.sh`
The startup script recreates the `Qwen/Qwen3.6-35B-A3B-FP8` symlink inside the
bundled ModelScope cache so the bundle remains portable across machines.
*************audio*************
# 前台运行,Ctrl+C 直接停止
./start_audio.sh
# 后台运行
nohup ./start_audio.sh > /dev/null 2>&1 &
# 后台运行时用 stop 停止
./stop_audio.sh
+123
View File
@@ -0,0 +1,123 @@
{%- set enable_thinking = enable_thinking | default(false) %}
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{%- if messages[0].content is string %}
{{- messages[0].content }}
{%- else %}
{%- for content in messages[0].content %}
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
{{- "<|vision_start|><|image_pad|><|vision_end|>" }}
{%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %}
{{- "<|audio_start|><|audio_pad|><|audio_end|>" }}
{%- elif content.type == 'video' or 'video' in content %}
{{- "<|vision_start|><|video_pad|><|vision_end|>" }}
{%- elif content.type == 'text' %}
{{- content.text }}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endif %}
{{- '\n\n' }}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{%- if messages[0].content is string %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- else %}
{%- for content in messages[0].content %}
{%- if content.type == 'image' or 'image' in content or 'image_url' in content %}
{{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }}
{%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %}
{{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }}
{%- elif content.type == 'video' or 'video' in content %}
{{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }}
{%- elif content.type == 'text' %}
{{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = namespace(text="") %}
{%- for mcontent in message.content %}
{%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %}
{%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %}
{%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %}
{%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %}
{%- elif mcontent.type == 'video' or 'video' in mcontent %}
{%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %}
{%- elif mcontent.type == 'text' %}
{%- set content.text = content.text~mcontent.text %}
{%- endif %}
{%- endfor %}
{%- set content = content.text %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = "" %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip("\n") + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}{{- '<think>\n\n</think>\n\n' }}{%- endif %}
{%- endif %}
+87
View File
@@ -0,0 +1,87 @@
from fastapi import FastAPI, UploadFile, Query
from fastapi.responses import Response
import io
import numpy as np
import soundfile as sf
from contextlib import asynccontextmanager
import torch
import tempfile
import os
# 直接指定本地模型文件夹(绝对路径)
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
ASR_MODEL_PATH = os.path.join(BASE_DIR, "modelscope", "hub", "models", "Qwen", "Qwen3-ASR-1___7B")
TTS_MODEL_PATH = os.path.join(BASE_DIR, "modelscope", "hub", "models", "OpenBMB", "VoxCPM2")
models = {}
@asynccontextmanager
async def lifespan(app: FastAPI):
from transformers import BitsAndBytesConfig
quantization_asr_config = BitsAndBytesConfig(load_in_8bit=True)
print(f"Loading ASR model from {ASR_MODEL_PATH} ...")
from qwen_asr import Qwen3ASRModel
models["asr"] = Qwen3ASRModel.from_pretrained(
ASR_MODEL_PATH,
dtype=torch.bfloat16,
device_map="cuda:0",
max_inference_batch_size=4,
attn_implementation="flash_attention_2",
max_new_tokens=512,
quantization_config=quantization_asr_config
)
from voxcpm import VoxCPM
models["tts"] = VoxCPM.from_pretrained(
TTS_MODEL_PATH,
load_denoiser=False,
)
models["tts_sample_rate"] = models["tts"].tts_model.sample_rate
print("ASR & TTS models loaded.")
yield
del models["asr"], models["tts"]
app = FastAPI(lifespan=lifespan)
@app.post("/asr")
async def speech_to_text(file: UploadFile):
# 保存上传文件为临时 WAV
suffix = os.path.splitext(file.filename)[-1] or ".wav"
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
tmp.write(await file.read())
tmp_path = tmp.name
try:
# 调用 transcribe,这里直接使用自动语言检测
results = models["asr"].transcribe(
audio=[tmp_path],
language=None, # 自动检测,或指定 ["Chinese", "English"]
return_time_stamps=False, # 如需时间戳可改为 True
)
# 取第一个结果(仅一个音频)
if results:
text = results[0].text
else:
text = ""
finally:
os.unlink(tmp_path) # 清理临时文件
return {"text": text}
@app.get("/tts")
async def text_to_speech(text: str = Query(...)):
with torch.no_grad():
wav = models["tts"].generate(
text=text,
cfg_value=2.0,
inference_timesteps=10,
)
buffer = io.BytesIO()
sf.write(buffer, wav, models["tts_sample_rate"], format="WAV")
return Response(buffer.getvalue(), media_type="audio/wav")
@app.get("/health")
async def health():
return {"status": "ok", "models": list(models.keys())}
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
set -euo pipefail
# 配置区域(可根据实际情况修改)
MODEL_BASE_PATH="${MODEL_BASE_PATH:-/mnt/Qwen3-Omni-30B-A3B-Instruct-NVFP4}"
THINKER_PATH="${MODEL_BASE_PATH}/thinker"
CONTAINER_NAME="${CONTAINER_NAME:-vllm-thinker}"
HOST_PORT="${HOST_PORT:-8091}"
GPU_DEVICES="${GPU_DEVICES:-all}" # 例如 "0,1" 或 "all"
TENSOR_PARALLEL_SIZE="${TENSOR_PARALLEL_SIZE:-1}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}"
VLLM_IMAGE="${VLLM_IMAGE:-vllm/vllm-openai:latest}"
# 检查模型路径是否存在
if [[ ! -d "$THINKER_PATH" ]]; then
echo "错误: 模型路径不存在: $THINKER_PATH"
echo "请设置环境变量 MODEL_BASE_PATH 指向正确的模型根目录(包含 thinker 子目录)"
exit 1
fi
# 检查 Docker 可用性
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
DOCKER_CMD="docker"
else
DOCKER_CMD="sudo docker"
fi
# 停止并移除可能已存在的同名容器
${DOCKER_CMD} rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
echo "启动 Thinker 服务..."
echo "模型路径: $THINKER_PATH"
echo "服务端口: $HOST_PORT"
echo "Tensor Parallel Size: $TENSOR_PARALLEL_SIZE"
# 启动容器
${DOCKER_CMD} run -d \
--name "${CONTAINER_NAME}" \
--gpus "${GPU_DEVICES}" \
--ipc=host \
--network host \
-v "${THINKER_PATH}:/model:ro" \
-e VLLM_USE_MODELSCOPE=false \
"${VLLM_IMAGE}" \
sh -c "pip install --no-cache-dir nvidia-modelopt && \
vllm serve /model \
--port ${HOST_PORT} \
--quantization modelopt_fp4 \
--dtype bfloat16 \
--max-model-len ${MAX_MODEL_LEN} \
--tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \
--trust-remote-code \
--enforce-eager"
# 等待服务启动
sleep 3
# 健康检查
echo "健康检查 (等待几秒后):"
echo " curl http://127.0.0.1:${HOST_PORT}/health"
echo "查看日志:"
echo " ${DOCKER_CMD} logs -f ${CONTAINER_NAME}"
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BUNDLE_DIR="${SCRIPT_DIR}"
VENV_DIR="${BUNDLE_DIR}/.venv_audio"
HOST_PORT="${HOST_PORT:-8100}"
# 激活虚拟环境
source "${VENV_DIR}/bin/activate"
# 检查端口占用
if lsof -i "tcp:${HOST_PORT}" -t >/dev/null 2>&1; then
echo "Port ${HOST_PORT} already in use" >&2
exit 1
fi
echo "Starting audio server on port ${HOST_PORT} ..."
cd "${BUNDLE_DIR}"
export MODELSCOPE_DOWNLOAD_MODE=local
export MODELSCOPE_HUB_ENABLE_FILE_CHECK=false
export MODELSCOPE_CACHE="${BUNDLE_DIR}/modelscope"
# 强制 HuggingFace transformers 离线
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
uvicorn speech_server:app --host 0.0.0.0 --port "${HOST_PORT}" --log-level info
View File
Whitespace-only changes.
+81
View File
@@ -0,0 +1,81 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BUNDLE_DIR="${SCRIPT_DIR}"
IMAGE_TAG="local/vllm-qwen36-nvfp4:v0.19.1-cu130"
IMAGE_TAR="${BUNDLE_DIR}/image/vllm-qwen36-nvfp4_v0.19.1-cu130.tar"
MODELSCOPE_DIR="${BUNDLE_DIR}/modelscope"
CHAT_TEMPLATE="${BUNDLE_DIR}/chat_template.jinja"
CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-nvfp4}"
HOST_PORT="${HOST_PORT:-8000}"
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
DOCKER=(docker)
else
DOCKER=(sudo docker)
fi
require_path() {
local path="$1"
if [[ ! -e "${path}" ]]; then
echo "Missing required path: ${path}" >&2
exit 1
fi
}
require_path "${IMAGE_TAR}"
require_path "${MODELSCOPE_DIR}"
require_path "${CHAT_TEMPLATE}"
mkdir -p "${MODELSCOPE_DIR}/hub/models/Qwen"
MODEL_REAL_DIR="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3___6-35B-A3B-FP8"
MODEL_LINK="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3.6-35B-A3B-FP8"
require_path "${MODEL_REAL_DIR}"
rm -f "${MODEL_LINK}"
ln -s "Qwen3___6-35B-A3B-FP8" "${MODEL_LINK}"
if ! "${DOCKER[@]}" image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
echo "Loading image ${IMAGE_TAG} from ${IMAGE_TAR}"
"${DOCKER[@]}" load -i "${IMAGE_TAR}"
fi
"${DOCKER[@]}" rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
echo "Starting container ${CONTAINER_NAME} on port ${HOST_PORT}"
"${DOCKER[@]}" run -d \
--name "${CONTAINER_NAME}" \
--gpus all \
--ipc=host \
--network host \
-e VLLM_USE_MODELSCOPE=true \
-v "${MODELSCOPE_DIR}:/root/.cache/modelscope" \
-v "${MODELSCOPE_DIR}:/home/liyh/.cache/modelscope" \
-v "${CHAT_TEMPLATE}:/root/chat_template.jinja:ro" \
"${IMAGE_TAG}" \
Qwen/Qwen3.6-35B-A3B-FP8 \
--port "${HOST_PORT}" \
--tensor-parallel-size 1 \
--max-model-len 131072 \
--reasoning-parser qwen3 \
--enable-auto-tool-choice \
--tool-call-parser qwen3_coder \
--api-key semi-ai.tech \
--enable-prefix-caching \
--max_num_batched_tokens 4096 \
--max_num_seqs 8 \
--kv-cache-dtype fp8_e4m3 \
--chat-template /root/chat_template.jinja \
--safetensors-load-strategy eager \
--gpu-memory-utilization 0.75\
"$@"
echo
echo "Health check:"
echo " curl http://127.0.0.1:${HOST_PORT}/health"
echo "Logs:"
echo " ${DOCKER[*]} logs -f ${CONTAINER_NAME}"
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
CONTAINER_NAME="${CONTAINER_NAME:-vllm-thinker}"
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
DOCKER_CMD="docker"
else
DOCKER_CMD="sudo docker"
fi
echo "停止并移除容器: $CONTAINER_NAME"
${DOCKER_CMD} rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
echo "已停止"
+14
View File
@@ -0,0 +1,14 @@
#!usrbinenv bash
set -euo pipefail
HOST_PORT=${HOST_PORT-8100}
PID=$(lsof -i tcp${HOST_PORT} -t 2devnull true)
if [[ -z ${PID} ]]; then
echo No process found on port ${HOST_PORT}
exit 0
fi
echo Stopping audio server (PID ${PID}) ...
kill ${PID}
View File
Whitespace-only changes.
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -euo pipefail
CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-nvfp4}"
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
DOCKER=(docker)
else
DOCKER=(sudo docker)
fi
"${DOCKER[@]}" rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
echo "Stopped ${CONTAINER_NAME}"
+148
View File
@@ -0,0 +1,148 @@
#!/usr/bin/env python3
"""
语音服务功能测试:ASR 识别 + TTS 合成
"""
import requests
import json
import os
import sys
import time
HOST = "http://127.0.0.1:8100"
def test_health():
"""1. 健康检查"""
print("=" * 50)
print("📋 健康检查")
print("=" * 50)
resp = requests.get(f"{HOST}/health")
print(f"状态码: {resp.status_code}")
print(f"模型列表: {resp.json()}")
assert resp.status_code == 200
assert "asr" in resp.json()["models"]
assert "tts" in resp.json()["models"]
print("✅ 健康检查通过\n")
def test_asr(audio_path=None):
"""2. 测试语音识别"""
print("=" * 50)
print("🎤 语音识别测试")
print("=" * 50)
if audio_path is None:
# 如果没有本地音频,下载一个公开示例
import tempfile
audio_path = tempfile.mktemp(suffix=".wav")
url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav"
print(f"下载测试音频: {url}")
import subprocess
subprocess.run(["wget", "-q", url, "-O", audio_path], check=True)
print(f"音频文件: {audio_path}")
with open(audio_path, "rb") as f:
resp = requests.post(f"{HOST}/asr", files={"file": f})
print(f"状态码: {resp.status_code}")
result = resp.json()
print(f"识别结果: {result.get('text', 'N/A')}")
assert resp.status_code == 200
assert len(result.get("text", "")) > 0
print("✅ 语音识别通过\n")
return result["text"]
def test_tts(text="(激动地说)你好世界,这是一段测试语音", output="test_output.wav"):
"""3. 测试语音合成"""
print("=" * 50)
print("🔊 语音合成测试")
print("=" * 50)
print(f"合成文本: {text}")
start_time = time.time()
resp = requests.get(f"{HOST}/tts", params={"text": text})
elapsed = time.time() - start_time
print(f"状态码: {resp.status_code}")
print(f"耗时: {elapsed:.2f}秒")
if resp.status_code == 200:
with open(output, "wb") as f:
f.write(resp.content)
file_size = os.path.getsize(output) / 1024 # KB
print(f"音频已保存: {output} ({file_size:.1f} KB)")
else:
print(f"错误: {resp.text}")
assert resp.status_code == 200
assert os.path.exists(output)
print("✅ 语音合成通过\n")
def test_asr_english():
"""4. 测试英文识别"""
print("=" * 50)
print("🌍 英文识别测试")
print("=" * 50)
import tempfile
audio_path = tempfile.mktemp(suffix=".wav")
url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav"
print(f"下载英文测试音频: {url}")
import subprocess
subprocess.run(["wget", "-q", url, "-O", audio_path], check=True)
with open(audio_path, "rb") as f:
resp = requests.post(f"{HOST}/asr", files={"file": f})
print(f"状态码: {resp.status_code}")
result = resp.json()
print(f"识别结果: {result.get('text', 'N/A')}")
assert resp.status_code == 200
assert len(result.get("text", "")) > 0
print("✅ 英文识别通过\n")
def test_tts_long_text():
"""5. 测试长文本合成"""
print("=" * 50)
print("📜 长文本合成测试")
print("=" * 50)
long_text = """人工智能是计算机科学的一个分支,它企图了解智能的实质,
并生产出一种新的能以人类智能相似的方式做出反应的智能机器,
该领域的研究包括机器人、语言识别、图像识别、自然语言处理和专家系统等。"""
print(f"文本长度: {len(long_text)} 字")
start_time = time.time()
resp = requests.get(f"{HOST}/tts", params={"text": long_text})
elapsed = time.time() - start_time
print(f"状态码: {resp.status_code}")
print(f"耗时: {elapsed:.2f}秒")
if resp.status_code == 200:
output = "test_long_output.wav"
with open(output, "wb") as f:
f.write(resp.content)
file_size = os.path.getsize(output) / 1024
print(f"音频已保存: {output} ({file_size:.1f} KB)")
assert resp.status_code == 200
print("✅ 长文本合成通过\n")
if __name__ == "__main__":
print("\n" + "🚀 开始测试语音服务".center(50, "="))
print(f"服务地址: {HOST}\n")
try:
test_health()
# 可选:用自己的音频文件测试
# test_asr("/path/to/your/audio.wav")
test_asr()
test_asr_english()
test_tts()
test_tts_long_text()
print("\n" + "🎉 所有测试通过!".center(50, "="))
except Exception as e:
print(f"\n❌ 测试失败: {e}")
sys.exit(1)