替换了llm基座，修改了调用接口，后续ASR采用vllm部署qwen3omni-30B-A3B可以做更细腻的音频识别（生成的是上文本描述），使用vllm部署qwen-asr-1.7B以满足实时场景（生成的是一对一字幕），TTS采用UV部署GPT-SoVITS轻量级TTS引擎可复制音色且300M的参数满足实时对话要求

.env

@@ -8,7 +8,9 @@ REDIS_URI=redis://localhost:6379/0
 # LLM_API_KEY='semi-ai.tech'
 # LLM_BASE_URL="http://192.168.137.96:8000/v1"
 # LLM_MODEL = "Qwen/Qwen3.6-35B-A3B-FP8"
-ASR_TTS_BASE_URL = "http://192.168.137.96:8100"
+# ASR_TTS_BASE_URL = "http://192.168.137.96:8100"
+TTS_BASE_URL = "http://192.168.137.96:8902"
+ASR_BASE_URL = "http://192.168.137.96:8901/v1"
 
 #6000pro：
 LLM_API_KEY='semi-ai.tech'

README.md

@@ -1,3 +1,8 @@
+cd .\Agent\
+git add .
+git commit -m "修改内容的说明"
+git push
+
 python -m venv myenv
 source myenv/bin/activate ||  myenv/Scripts/activate
 python -m pip install --upgrade pip

src/agent/llm.py

@@ -15,7 +15,7 @@ from langchain_core.outputs import ChatResult, ChatGeneration
 from langchain_openai import ChatOpenAI
 
 from agent.utils.image_utils import image_bytes_to_openai_image_url_part
-from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, ASR_TTS_BASE_URL
+from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, TTS_BASE_URL, ASR_BASE_URL
 from agent.utils.agent_utils import assistant_text, normalize_user_text_content
 from agent.utils.image_utils import (
     image_path_to_openai_image_url_part,
@@ -107,13 +107,33 @@ class ChatQwen(ChatOpenAI):
         wav_path = self._convert_audio_to_wav(audio_path)
         cleanup_wav = wav_path != audio_path
         try:
-            with open(wav_path, "rb") as f:
+            # 构造 file:// URL（vLLM 要求）
-                resp = requests.post(
+            abs_path = os.path.abspath(wav_path).replace("\\", "/")
-                    f"{ASR_TTS_BASE_URL}/asr",
+            audio_url = f"file://{abs_path}"
-                    files={"file": (os.path.basename(wav_path), f, "audio/wav")},
+
-                )
+            payload = {
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "audio_url", "audio_url": {"url": audio_url}}
+                        ]
+                    }
+                ]
+            }
+            # ASR_BASE_URL 末尾可能已带 /v1，统一拼接 /chat/completions
+            endpoint = f"{ASR_BASE_URL.rstrip('/')}/chat/completions"
+            resp = requests.post(
+                endpoint,
+                headers={"Content-Type": "application/json"},
+                json=payload,
+                timeout=120
+            )
             resp.raise_for_status()
-            return (resp.json().get("text") or "").strip()
+            result = resp.json()
+            # 提取 assistant 回复文本（音频描述/转录）
+            content = result["choices"][0]["message"]["content"]
+            return content.strip()
         finally:
             if cleanup_wav and os.path.exists(wav_path):
                 os.remove(wav_path)
@@ -174,7 +194,7 @@ class ChatQwen(ChatOpenAI):
         return new_messages
 
     def _text_to_audio(self, text: str) -> bytes:
-        resp = requests.get(f"{ASR_TTS_BASE_URL}/tts", params={"text": text})
+        resp = requests.get(f"{TTS_BASE_URL}/tts", params={"text": text})
         resp.raise_for_status()
         return resp.content
 

src/agent/utils/env_utils.py

@@ -25,6 +25,8 @@ LLM_REQUESTS_PER_SECOND = _get_env("LLM_REQUESTS_PER_SECOND", default="10", cast
 LLM_MAX_BUCKET_SIZE = _get_env("LLM_MAX_BUCKET_SIZE", default="10", cast=int)
 LLM_CHECK_EVERY_N_SECONDS = _get_env("LLM_CHECK_EVERY_N_SECONDS", default="0.1", cast=float)
 ASR_TTS_BASE_URL = _get_env("ASR_TTS_BASE_URL")
+TTS_BASE_URL = _get_env("TTS_BASE_URL")
+ASR_BASE_URL = _get_env("ASR_BASE_URL")
 
 BROWSER_MAP: Dict[str, Tuple[str, str]] = {
     "edge": ("microsoft-edge", "Microsoft Edge"),