From 64cabe63d7063896b79ffea86db3adfb7a3c72ba Mon Sep 17 00:00:00 2001 From: minorQAQ <202200120184@mail.sdu.edu.cn> Date: Sat, 16 May 2026 00:08:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E4=BA=86llm=E5=9F=BA?= =?UTF-8?q?=E5=BA=A7=EF=BC=8C=E4=BF=AE=E6=94=B9=E4=BA=86=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=EF=BC=8C=E5=90=8E=E7=BB=ADASR=E9=87=87?= =?UTF-8?q?=E7=94=A8vllm=E9=83=A8=E7=BD=B2qwen3omni-30B-A3B=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E5=81=9A=E6=9B=B4=E7=BB=86=E8=85=BB=E7=9A=84=E9=9F=B3?= =?UTF-8?q?=E9=A2=91=E8=AF=86=E5=88=AB=EF=BC=88=E7=94=9F=E6=88=90=E7=9A=84?= =?UTF-8?q?=E6=98=AF=E4=B8=8A=E6=96=87=E6=9C=AC=E6=8F=8F=E8=BF=B0=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E4=BD=BF=E7=94=A8vllm=E9=83=A8=E7=BD=B2qwen-asr-1.7B?= =?UTF-8?q?=E4=BB=A5=E6=BB=A1=E8=B6=B3=E5=AE=9E=E6=97=B6=E5=9C=BA=E6=99=AF?= =?UTF-8?q?=EF=BC=88=E7=94=9F=E6=88=90=E7=9A=84=E6=98=AF=E4=B8=80=E5=AF=B9?= =?UTF-8?q?=E4=B8=80=E5=AD=97=E5=B9=95=EF=BC=89=EF=BC=8CTTS=E9=87=87?= =?UTF-8?q?=E7=94=A8UV=E9=83=A8=E7=BD=B2GPT-SoVITS=E8=BD=BB=E9=87=8F?= =?UTF-8?q?=E7=BA=A7TTS=E5=BC=95=E6=93=8E=E5=8F=AF=E5=A4=8D=E5=88=B6?= =?UTF-8?q?=E9=9F=B3=E8=89=B2=E4=B8=94300M=E7=9A=84=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E6=BB=A1=E8=B6=B3=E5=AE=9E=E6=97=B6=E5=AF=B9=E8=AF=9D=E8=A6=81?= =?UTF-8?q?=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env | 4 +- README.md | 5 +++ ...a => 60a5fff67eaf461b979d76449a1a6224.wav} | Bin src/agent/llm.py | 36 ++++++++++++++---- src/agent/utils/env_utils.py | 2 + 5 files changed, 38 insertions(+), 9 deletions(-) rename src/agent/history/multi_inputs/20260514_222254/{60a5fff67eaf461b979d76449a1a6224.m4a => 60a5fff67eaf461b979d76449a1a6224.wav} (100%) diff --git a/.env b/.env index 01284ab..04a82ab 100644 --- a/.env +++ b/.env @@ -8,7 +8,9 @@ REDIS_URI=redis://localhost:6379/0 # LLM_API_KEY='semi-ai.tech' # LLM_BASE_URL="http://192.168.137.96:8000/v1" # LLM_MODEL = "Qwen/Qwen3.6-35B-A3B-FP8" -ASR_TTS_BASE_URL = "http://192.168.137.96:8100" +# ASR_TTS_BASE_URL = "http://192.168.137.96:8100" +TTS_BASE_URL = "http://192.168.137.96:8902" +ASR_BASE_URL = "http://192.168.137.96:8901/v1" #6000pro： LLM_API_KEY='semi-ai.tech' diff --git a/README.md b/README.md index 27eb401..d691334 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +cd .\Agent\ +git add . +git commit -m "修改内容的说明" +git push + python -m venv myenv source myenv/bin/activate || myenv/Scripts/activate python -m pip install --upgrade pip diff --git a/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a b/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.wav similarity index 100% rename from src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a rename to src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.wav diff --git a/src/agent/llm.py b/src/agent/llm.py index 764f71a..c4574e9 100644 --- a/src/agent/llm.py +++ b/src/agent/llm.py @@ -15,7 +15,7 @@ from langchain_core.outputs import ChatResult, ChatGeneration from langchain_openai import ChatOpenAI from agent.utils.image_utils import image_bytes_to_openai_image_url_part -from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, ASR_TTS_BASE_URL +from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, TTS_BASE_URL, ASR_BASE_URL from agent.utils.agent_utils import assistant_text, normalize_user_text_content from agent.utils.image_utils import ( image_path_to_openai_image_url_part, @@ -107,13 +107,33 @@ class ChatQwen(ChatOpenAI): wav_path = self._convert_audio_to_wav(audio_path) cleanup_wav = wav_path != audio_path try: - with open(wav_path, "rb") as f: - resp = requests.post( - f"{ASR_TTS_BASE_URL}/asr", - files={"file": (os.path.basename(wav_path), f, "audio/wav")}, - ) + # 构造 file:// URL（vLLM 要求） + abs_path = os.path.abspath(wav_path).replace("\\", "/") + audio_url = f"file://{abs_path}" + + payload = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "audio_url", "audio_url": {"url": audio_url}} + ] + } + ] + } + # ASR_BASE_URL 末尾可能已带 /v1，统一拼接 /chat/completions + endpoint = f"{ASR_BASE_URL.rstrip('/')}/chat/completions" + resp = requests.post( + endpoint, + headers={"Content-Type": "application/json"}, + json=payload, + timeout=120 + ) resp.raise_for_status() - return (resp.json().get("text") or "").strip() + result = resp.json() + # 提取 assistant 回复文本（音频描述/转录） + content = result["choices"][0]["message"]["content"] + return content.strip() finally: if cleanup_wav and os.path.exists(wav_path): os.remove(wav_path) @@ -174,7 +194,7 @@ class ChatQwen(ChatOpenAI): return new_messages def _text_to_audio(self, text: str) -> bytes: - resp = requests.get(f"{ASR_TTS_BASE_URL}/tts", params={"text": text}) + resp = requests.get(f"{TTS_BASE_URL}/tts", params={"text": text}) resp.raise_for_status() return resp.content diff --git a/src/agent/utils/env_utils.py b/src/agent/utils/env_utils.py index 7ccca83..3292bb2 100644 --- a/src/agent/utils/env_utils.py +++ b/src/agent/utils/env_utils.py @@ -25,6 +25,8 @@ LLM_REQUESTS_PER_SECOND = _get_env("LLM_REQUESTS_PER_SECOND", default="10", cast LLM_MAX_BUCKET_SIZE = _get_env("LLM_MAX_BUCKET_SIZE", default="10", cast=int) LLM_CHECK_EVERY_N_SECONDS = _get_env("LLM_CHECK_EVERY_N_SECONDS", default="0.1", cast=float) ASR_TTS_BASE_URL = _get_env("ASR_TTS_BASE_URL") +TTS_BASE_URL = _get_env("TTS_BASE_URL") +ASR_BASE_URL = _get_env("ASR_BASE_URL") BROWSER_MAP: Dict[str, Tuple[str, str]] = { "edge": ("microsoft-edge", "Microsoft Edge"),