diff --git a/.env b/.env index 01c51a8..3db4a82 100644 --- a/.env +++ b/.env @@ -16,8 +16,7 @@ RAG_BASE_URL = "http://192.168.137.96:8903" #6000pro： LLM_API_KEY='semi-ai.tech' LLM_BASE_URL="http://semi-ai.tech:40056/v1" -LLM_MODEL = "Intel/Qwen3.6-27B-int4-AutoRound" - +LLM_MODEL = "Intel/Qwen3.6-35B-A3B-int4-mixed-AutoRound" #A100： # LLM_API_KEY = 'sk-kGhWAQTqLk9ECsmIDb758a456bD34e92B9771dF2E36432Fb' @@ -27,3 +26,12 @@ LLM_MODEL = "Intel/Qwen3.6-27B-int4-AutoRound" # LLM_REQUESTS_PER_SECOND = 10.0 # LLM_MAX_BUCKET_SIZE = 10 # LLM_CHECK_EVERY_N_SECONDS = 0.1 + +#context: +LLM_CONTEXT_WINDOW=32768 +MINI_COMPRESS_RATE=0.4 +HARD_COMPRESS_RATE=0.6 +CURRENT_TURNS=4 +IMG_SIZE=768 +RAG_CHUNK_SIZE=500 +RAG_CHUNK_OVERLAP=50 \ No newline at end of file diff --git a/README.md b/README.md index 2aa83c7..731cfdf 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,36 @@ -# 进入项目目录并提交代码（Windows 下路径为 .\Agent\） +# Agent Web Chat + +基于 LangGraph 的多模态对话 Agent，支持文本、图片、音频输入，具备屏幕截图、浏览器控制、RAG 知识库、数据库查询等工具调用能力。 + +## 快速开始 + +### 1. 克隆仓库并进入项目目录 + +```bash +git clone cd Agent -git add . -git commit -m "修改内容的说明" -git push -# 创建并激活虚拟环境（Linux/macOS 用 source，Windows 用 myenv\Scripts\activate） +Windows: +bash python -m venv myenv -source myenv/bin/activate || myenv/Scripts/activate +.\myenv\Scripts\activate +macOS / Linux: +bash +python -m venv myenv +source myenv/bin/activate -# 升级 pip 并安装核心依赖 -python -m pip install --upgrade pip -pip install -U langchain langchain-openai python-dotenv - -# 以可编辑模式安装当前项目（配置见 pyproject.toml） +pip install --upgrade pip +pip install -r requirements.txt pip install -e . -# 安装 LangGraph CLI（开发模式） -pip install "langgraph-cli[inmem]" - -# 可选：安装数据库相关依赖（psycopg2-binary 用于 PostgreSQL） -pip install sqlalchemy pymysql loguru psycopg2-binary - -# 环境变量配置 -# Copy .env.example to .env and fill in the required values before running the app. -cp .env.example .env - -# 必须填写的变量： -# .env.example +gedit .env +必须填写的变量： +LLM_BASE_URL – 大模型 API 基础地址 +LLM_API_KEY – API 密钥 +LLM_MODEL – 模型名称（如 Qwen/Qwen2.5-72B-Instruct） +TTS_BASE_URL – 文本转语音服务地址（可选，若不使用音频回复可不填） +ASR_BASE_URL – 语音识别服务地址（可选） +RAG_BASE_URL – RAG 嵌入与重排序服务地址（可选） # 启动 Web 对话服务（推荐） # 确保在 Agent 目录下，且 PYTHONPATH 包含 src 目录 @@ -35,8 +39,11 @@ export PYTHONPATH=src:$PYTHONPATH # Linux/macOS # 或 Windows PowerShell: $env:PYTHONPATH="src;$env:PYTHONPATH" uvicorn agent.web.server:app --host 127.0.0.1 --port 8765 -# API 为两阶段调用：POST /api/chat/start（落盘用户消息） + POST /api/chat/complete（推理） -# 前端已按此顺序调用。 - # （可选）通过 Cloudflare Tunnel 暴露公网 -cloudflared tunnel --url http://localhost:8765 \ No newline at end of file +cloudflared tunnel --url http://localhost:8765 + +# 进入项目目录并提交代码（Windows 下路径为 .\Agent\） +cd Agent +git add . +git commit -m "修改内容的说明" +git push diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..cd2918c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,43 @@ +# Core LangChain & LangGraph +langchain>=0.1.0 +langchain-core>=0.1.0 +langchain-community>=0.1.0 +langchain-openai>=0.2.0 +langgraph>=1.0.0 +openai>=1.0.0 + +# Web framework & server +fastapi>=0.115.0 +uvicorn[standard]>=0.32.0 +python-multipart>=0.0.9 + +# Database & vector store +sqlalchemy>=2.0.0 +psycopg2-binary>=2.9.0 +chromadb>=0.4.0 + +# Audio / TTS / ASR +pyaudio>=0.2.11 +wave +pydub>=0.25.1 # for audio format conversion +requests>=2.31.0 + +# Screen capture & browser control +pyautogui>=0.9.54 +webbrowser # built-in, no need to list + +# Image processing +Pillow>=10.0.0 + +# Document parsing (RAG) +pandas>=2.0.0 +python-docx>=1.0.0 +pdfplumber>=0.10.0 + +# Utilities +python-dotenv>=1.0.1 +loguru>=0.7.0 +pydantic>=2.0.0 + +# Optional but often needed +typing-extensions>=4.0.0 \ No newline at end of file diff --git a/src/__pycache__/__init__.cpython-312.pyc b/src/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index cc563b1..0000000 Binary files a/src/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agent/__pycache__/__init__.cpython-312.pyc b/src/agent/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 54b9c69..0000000 Binary files a/src/agent/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agent/__pycache__/chatbot_agent.cpython-312.pyc b/src/agent/__pycache__/chatbot_agent.cpython-312.pyc deleted file mode 100644 index b2353bc..0000000 Binary files a/src/agent/__pycache__/chatbot_agent.cpython-312.pyc and /dev/null differ diff --git a/src/agent/__pycache__/llm.cpython-312.pyc b/src/agent/__pycache__/llm.cpython-312.pyc deleted file mode 100644 index d75f5e2..0000000 Binary files a/src/agent/__pycache__/llm.cpython-312.pyc and /dev/null differ diff --git a/src/agent/chatbot_agent.py b/src/agent/chatbot_agent.py index bd86810..9bd627f 100644 --- a/src/agent/chatbot_agent.py +++ b/src/agent/chatbot_agent.py @@ -1,101 +1,45 @@ -from typing import Any, cast, Dict +# agent/chatbot_agent.py +from typing import Any, Dict -from langchain_classic.agents import AgentExecutor, create_tool_calling_agent -from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder -from langchain_core.runnables import RunnablePassthrough, RunnableWithMessageHistory -from langchain_core.messages import HumanMessage +from langchain_core.messages import HumanMessage, SystemMessage, AIMessage +from langchain.agents import create_agent from agent.llm import Qwen36_35B_A3B_FP8_LLM from agent.tools import get_tools -from agent.utils.agent_utils import assistant_text, get_history, normalize_user_text_content, summarize_history +from agent.utils.agent_utils import ( + assistant_text, + get_history, + summarizer, # 使用新的上下文感知构建器 + get_session_meta, + update_session_meta, +) from agent.web.ui_session import ( - collect_media_paths_and_query_text, rewrite_user_message_files_to_session_dir, session_dir, sync_manifest_from_chat_history, ) from agent.memory.system_prompt import MAIN_SYSTEM_PROMPT -# ---------- 1. 提示模板---------- -prompt = ChatPromptTemplate.from_messages([ - ("system", "{system_message}"), - ("placeholder", "{chat_history}"), - MessagesPlaceholder(variable_name="human_message"), - ("placeholder", "{agent_scratchpad}"), -]) - +# ---------- 1. 工具 ---------- tools = get_tools() -# ---------- 2. 创建工具调用 agent ---------- -tool_calling_agent = create_tool_calling_agent( - llm=Qwen36_35B_A3B_FP8_LLM.bind(tool_choice="auto"), +# ---------- 2. 创建 ReAct agent（LangGraph）---------- +agent_graph = create_agent( + model=Qwen36_35B_A3B_FP8_LLM, tools=tools, - prompt=prompt, + name="main" ) -# ---------- 3. agent executor ---------- -agent_executor = AgentExecutor( - agent=tool_calling_agent, - tools=tools, - verbose=True, - handle_parsing_errors=True, - stream_runnable=False, -) +# ---------- 3. 辅助函数：保存消息到数据库 ---------- +def save_messages_to_history(session_id: str, user_msg: HumanMessage, assistant_msg: AIMessage) -> None: + """将用户消息和助手消息追加到 SQLite 历史记录中""" + if not session_id: + return + history = get_history(session_id) + history.add_message(user_msg) + history.add_message(assistant_msg) -# ---------- 4. 带历史记录的链 ---------- -agent_with_history = RunnableWithMessageHistory( - runnable=agent_executor, - get_session_history=get_history, - input_messages_key="human_message", - history_messages_key="chat_history", -) - -# ---------- 5. 注入摘要和系统提示 ---------- -agent_with_history = ( - RunnablePassthrough.assign( - # ✅ 从输入数据中的 agent_config 获取配置 - summarize_result=lambda x: summarize_history(x["input"], x["agent_config"]) - ) - | RunnablePassthrough.assign( - input=lambda x: x["input"], - chat_history=lambda x: x["summarize_result"]["chat_history"], - system_message=lambda x: ( - MAIN_SYSTEM_PROMPT - + ( - "\n\n【此前对话摘要】\n" + x["summarize_result"]["summary"] - if x["summarize_result"]["summary"] - else "" - ) - ), - human_message=lambda x: [HumanMessage(content=x["input"])], - ) - | agent_with_history -) - - -# ---------- 6. 辅助函数：将前端 chat_history 转为 LangChain 消息列表 ---------- -def _extract_user_input_parts(user_msg: Dict[str, Any]) -> tuple[list[str], list[str], list[dict[str, Any]] | str]: - image_paths, audio_paths, query_text = collect_media_paths_and_query_text(user_msg) - - content_parts: list[dict[str, Any]] = [] - for img_path in image_paths: - content_parts.append({"type": "image_url", "image_url": {"url": img_path}}) - - for audio_path in audio_paths: - content_parts.append({ - "type": "audio", - "audio_path": audio_path, - }) - if query_text: - content_parts.append({"type": "text", "text": query_text}) - - if content_parts: - if len(content_parts) == 1 and content_parts[0].get("type") == "text": - return image_paths, audio_paths, content_parts[0]["text"] - return image_paths, audio_paths, content_parts - return image_paths, audio_paths, normalize_user_text_content(user_msg.get("content", query_text)) - -# ---------- 7. 核心执行函数 ---------- +# ---------- 4. 核心执行函数 ---------- def execute_agent( chat_history: list[dict[str, Any]], session_id: str = "test", @@ -106,34 +50,65 @@ def execute_agent( if not chat_history or chat_history[-1].get("role") != "user": return chat_history + # 上传文件处理 if session_id: rewrite_user_message_files_to_session_dir(chat_history[-1], session_id) - _image_paths, _audio_paths, user_content = _extract_user_input_parts(chat_history[-1]) + # 直接使用 chat_history 中已经正确格式化的 content（由 _build_user_chat_content 生成） + user_content = chat_history[-1]["content"] - # 构建运行时配置 - config = { - "configurable": { - "session_id": session_id, - }, + # 动态上下文感知构建消息 + messages, context_exceeded = summarizer( + session_id=session_id, + agent_name="main", # 固定 agent 名称，可扩展 + system_prompt=MAIN_SYSTEM_PROMPT, + user_content=user_content, + llm=Qwen36_35B_A3B_FP8_LLM, + ) + + if context_exceeded: + # 上下文已满，返回提示信息 + chat_history.append({ + "role": "assistant", + "content": "上下文已满，请新建会话继续对话。", + }) + if session_id: + sync_manifest_from_chat_history(session_id, chat_history, "") + return chat_history + + # 调用 graph 的配置 + graph_config = { + "configurable": {"session_id": session_id}, "metadata": { "output_type": output_type, "audio_output_dir": session_dir(session_id) if session_id else None, }, - "llm": Qwen36_35B_A3B_FP8_LLM, - } - - invoke_input = { - "input": user_content, - "agent_config": config, } try: - result = cast(dict[str, Any], agent_with_history.invoke(invoke_input, config)) + # 调用 LangGraph agent + result = agent_graph.invoke({"messages": messages}, config=graph_config) - text_content = assistant_text(result.get("output", "")) + # 从最终消息中提取文本回复 + all_messages = result["messages"] + assistant_msg = all_messages[-1] if all_messages else AIMessage(content="") + text_content = assistant_text(assistant_msg.content) - output_type_from_config = config.get("metadata", {}).get("output_type", "text") + # 提取 token 使用量并更新到 meta + usage = assistant_msg.usage_metadata if hasattr(assistant_msg, 'usage_metadata') else {} + total_tokens = usage.get('total_tokens', 0) if isinstance(usage, dict) else 0 + if total_tokens > 0: + meta = get_session_meta(session_id, "main") + update_session_meta( + session_id=session_id, + agent_name="main", + cursor_index=meta["cursor_index"], + compressed_content=meta["compressed_content"], + last_total_tokens=total_tokens, + ) + + # 处理 TTS 音频数据 + output_type_from_config = graph_config.get("metadata", {}).get("output_type", "text") audio_data = Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs or {} if audio_data.get("output_type") != output_type_from_config: audio_data = {} @@ -144,6 +119,7 @@ def execute_agent( "audio": audio_data.get("audio"), } + # 根据输出类型构建前端 content if output_type_from_config == "audio" and assistant_meta.get("audio"): assistant_content = [] elif output_type_from_config == "both" and assistant_meta.get("audio"): @@ -151,28 +127,32 @@ def execute_agent( else: assistant_content = text_content if text_content else "（空回复）" + # 追加到返回的 chat_history chat_history.append({ "role": "assistant", "content": assistant_content, "meta": assistant_meta, }) - # 用完后清空，避免影响后续无音频的请求 + # 持久化历史消息 + save_messages_to_history( + session_id, + HumanMessage(content=user_content), + AIMessage(content=text_content), + ) + + # 清空音频缓存 Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs = None except Exception as e: - # 异常处理保持不变 import traceback print("[ERROR] 调用模型时出错:") traceback.print_exc() err_text = "抱歉，我遇到了一些问题，请稍后再试。" chat_history.append({"role": "assistant", "content": err_text}) + # 同步文件清单 if session_id: sync_manifest_from_chat_history(session_id, chat_history, "") - return chat_history - - -if __name__ == "__main__": - pass \ No newline at end of file + return chat_history \ No newline at end of file diff --git a/src/agent/history/data_base/chat_history.db b/src/agent/history/data_base/chat_history.db deleted file mode 100644 index e5756b6..0000000 Binary files a/src/agent/history/data_base/chat_history.db and /dev/null differ diff --git a/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.wav b/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.wav deleted file mode 100644 index ba3c2c7..0000000 Binary files a/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.wav and /dev/null differ diff --git a/src/agent/history/multi_inputs/20260514_222254/assistant_tts_222333_0.wav b/src/agent/history/multi_inputs/20260514_222254/assistant_tts_222333_0.wav deleted file mode 100644 index 196fc5c..0000000 Binary files a/src/agent/history/multi_inputs/20260514_222254/assistant_tts_222333_0.wav and /dev/null differ diff --git a/src/agent/history/multi_inputs/20260514_222254/b50c460b07654d568e897d92c776675f.jpg b/src/agent/history/multi_inputs/20260514_222254/b50c460b07654d568e897d92c776675f.jpg deleted file mode 100644 index f8f53af..0000000 Binary files a/src/agent/history/multi_inputs/20260514_222254/b50c460b07654d568e897d92c776675f.jpg and /dev/null differ diff --git a/src/agent/history/multi_inputs/20260514_222254/ui_messages.json b/src/agent/history/multi_inputs/20260514_222254/ui_messages.json deleted file mode 100644 index 7a6bd5c..0000000 --- a/src/agent/history/multi_inputs/20260514_222254/ui_messages.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "version": 1, - "turns": [ - { - "user": { - "parts": [ - { - "kind": "image_url", - "relpath": "20260514_222254/b50c460b07654d568e897d92c776675f.jpg" - }, - { - "kind": "audio", - "relpath": "20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a" - }, - { - "kind": "text", - "text": "请直接告诉我数量。" - } - ], - "image_description": "" - }, - "assistant": { - "text": "图片中有两只猫。", - "output_type": "both", - "autoplay": false, - "audio_duration_seconds": 2.08, - "audio_relpath": "20260514_222254/assistant_tts_222333_0.wav" - } - } - ] -} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260515_130220/ui_messages.json b/src/agent/history/multi_inputs/20260515_130220/ui_messages.json deleted file mode 100644 index 3c3bad0..0000000 --- a/src/agent/history/multi_inputs/20260515_130220/ui_messages.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "version": 2, - "turns": [ - { - "user": { - "content": [ - { - "type": "text", - "text": "你好" - } - ] - }, - "assistant": { - "text": "你好！很高兴为您服务。请问有什么我可以帮您的吗？", - "output_type": "text", - "autoplay": false, - "audio_duration_seconds": null, - "audio_relpath": null - } - }, - { - "user": { - "content": [ - { - "type": "text", - "text": "现在几点了" - } - ] - }, - "assistant": { - "text": "现在是2026年5月15日 13:02。", - "output_type": "text", - "autoplay": false, - "audio_duration_seconds": null, - "audio_relpath": null - } - } - ] -} \ No newline at end of file diff --git a/src/agent/llm.py b/src/agent/llm.py index c4574e9..7c31435 100644 --- a/src/agent/llm.py +++ b/src/agent/llm.py @@ -1,3 +1,4 @@ +# agent/llm.py import io import os import tempfile @@ -16,7 +17,7 @@ from langchain_openai import ChatOpenAI from agent.utils.image_utils import image_bytes_to_openai_image_url_part from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, TTS_BASE_URL, ASR_BASE_URL -from agent.utils.agent_utils import assistant_text, normalize_user_text_content +from agent.utils.agent_utils import assistant_text from agent.utils.image_utils import ( image_path_to_openai_image_url_part, image_bytes_to_openai_image_url_part, @@ -104,24 +105,31 @@ class ChatQwen(ChatOpenAI): def _audio_to_text(self, audio_path: str) -> str: if not os.path.exists(audio_path): raise FileNotFoundError(f"音频文件不存在: {audio_path}") + wav_path = self._convert_audio_to_wav(audio_path) - cleanup_wav = wav_path != audio_path + cleanup_wav = (wav_path != audio_path) + try: - # 构造 file:// URL（vLLM 要求） - abs_path = os.path.abspath(wav_path).replace("\\", "/") - audio_url = f"file://{abs_path}" - + # 读取音频文件二进制内容 + with open(wav_path, "rb") as f: + audio_bytes = f.read() + + # 构建 Data URL + import base64 + b64 = base64.b64encode(audio_bytes).decode("ascii") + mime_type = "audio/wav" # 我们已经转换为 wav + data_url = f"data:{mime_type};base64,{b64}" + payload = { "messages": [ { "role": "user", "content": [ - {"type": "audio_url", "audio_url": {"url": audio_url}} + {"type": "audio_url", "audio_url": {"url": data_url}} ] } ] } - # ASR_BASE_URL 末尾可能已带 /v1，统一拼接 /chat/completions endpoint = f"{ASR_BASE_URL.rstrip('/')}/chat/completions" resp = requests.post( endpoint, @@ -131,7 +139,6 @@ class ChatQwen(ChatOpenAI): ) resp.raise_for_status() result = resp.json() - # 提取 assistant 回复文本（音频描述/转录） content = result["choices"][0]["message"]["content"] return content.strip() finally: @@ -151,7 +158,12 @@ class ChatQwen(ChatOpenAI): except (json.JSONDecodeError, ValueError): pass if isinstance(msg, HumanMessage) and isinstance(msg.content, dict): - msg = HumanMessage(content=normalize_user_text_content(msg.content), additional_kwargs=msg.additional_kwargs) + # 尝试从 dict 中提取文本，否则序列化为字符串 + text = msg.content.get("text") or msg.content.get("content") or "" + if text: + msg = HumanMessage(content=str(text), additional_kwargs=msg.additional_kwargs) + else: + msg = HumanMessage(content=json.dumps(msg.content, ensure_ascii=False), additional_kwargs=msg.additional_kwargs) elif isinstance(msg, HumanMessage) and isinstance(msg.content, list): new_content: List[Dict[str, Any] | str] = [] for part in msg.content: @@ -179,7 +191,7 @@ class ChatQwen(ChatOpenAI): else: new_content.append({"type": "text", "text": f"[用户上传了附件文件: {os.path.basename(file_path)}]"}) else: - text = normalize_user_text_content(part) + text = assistant_text(part) if text: new_content.append({"type": "text", "text": text}) elif isinstance(part, str): @@ -266,7 +278,7 @@ class ChatQwen(ChatOpenAI): output_type = metadata.get("output_type", self.output_type) audio_output_dir = metadata.get("audio_output_dir") - # ✅ 移除自定义参数，防止它们传入 OpenAI API + # 移除自定义参数，防止它们传入 OpenAI API kwargs.pop("output_type", None) kwargs.pop("audio_output_dir", None) @@ -354,3 +366,37 @@ class ChatQwen(ChatOpenAI): Qwen36_35B_A3B_FP8_LLM = ChatQwen() + +if __name__ == "__main__": + import sys + + # 使用原始字符串避免转义问题 + TEST_IMAGE_PATH = r"C:\Users\86166\Desktop\image_raw\4.jpg" + + if not os.path.exists(TEST_IMAGE_PATH): + print(f"❌ 图片不存在: {TEST_IMAGE_PATH}") + sys.exit(1) + + # 测试1: 纯图片识别 + print("=" * 50) + print("测试1: 纯图片识别（无文本）") + print("=" * 50) + result1 = Qwen36_35B_A3B_FP8_LLM.invoke([ + HumanMessage(content=[{"type": "image_url", "image_url": {"url": TEST_IMAGE_PATH}}]) + ]) + print(f"返回类型: {type(result1)}") + print(f"返回内容: {result1.content}") + print(f"所有属性: {[attr for attr in dir(result1) if not attr.startswith('_')]}") + + # 测试2: 图片 + 文本 + print("\n" + "=" * 50) + print("测试2: 图文混合输入") + print("=" * 50) + result2 = Qwen36_35B_A3B_FP8_LLM.invoke([ + HumanMessage(content=[ + {"type": "image_url", "image_url": {"url": TEST_IMAGE_PATH}}, + {"type": "text", "text": "这张图片里有什么？请详细描述"} + ]) + ]) + print(f"回答: {result2.content}") + diff --git a/src/agent/memory/__pycache__/__init__.cpython-312.pyc b/src/agent/memory/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 318af58..0000000 Binary files a/src/agent/memory/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agent/memory/__pycache__/system_prompt.cpython-312.pyc b/src/agent/memory/__pycache__/system_prompt.cpython-312.pyc deleted file mode 100644 index 0bfd68a..0000000 Binary files a/src/agent/memory/__pycache__/system_prompt.cpython-312.pyc and /dev/null differ diff --git a/src/agent/tools/__init__.py b/src/agent/tools/__init__.py index 4ea8e30..e36bd43 100644 --- a/src/agent/tools/__init__.py +++ b/src/agent/tools/__init__.py @@ -1,4 +1,5 @@ -from .tool_main import * +from .tool import * +from .tool_rag import RAGTool # from .tool_database_class import * # from utils.database_utils import PostgreSQL_Manager @@ -20,12 +21,12 @@ from .tool_main import * # def get_sql_tools(): # return _all_database_tools - _all_tools = [ Websearchclass(), DateQueryClass(), ScreenCaptureDescribeClass(), - BrowserControlClass() + BrowserControlClass(), + RAGTool(), ] def get_tools(): diff --git a/src/agent/tools/__pycache__/__init__.cpython-312.pyc b/src/agent/tools/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 3950798..0000000 Binary files a/src/agent/tools/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agent/tools/__pycache__/tool.cpython-312.pyc b/src/agent/tools/__pycache__/tool.cpython-312.pyc deleted file mode 100644 index e2dbc5e..0000000 Binary files a/src/agent/tools/__pycache__/tool.cpython-312.pyc and /dev/null differ diff --git a/src/agent/tools/__pycache__/tool_main.cpython-312.pyc b/src/agent/tools/__pycache__/tool_main.cpython-312.pyc deleted file mode 100644 index e992f31..0000000 Binary files a/src/agent/tools/__pycache__/tool_main.cpython-312.pyc and /dev/null differ diff --git a/src/agent/tools/tool.py b/src/agent/tools/tool.py index af820a0..ff2e9f5 100644 --- a/src/agent/tools/tool.py +++ b/src/agent/tools/tool.py @@ -1,54 +1,252 @@ -from langchain.tools import tool +from __future__ import annotations -@tool("web_search") -def web_search(query: str) -> str: - """ - 一个用于执行网络搜索的工具。它接收一个查询语句作为输入，并返回 Google 的搜索结果。 - - Args: - query (str): 搜索查询内容 - - Returns: - str: 格式化后的搜索结果 - """ - try: +import io +from typing import Optional, Literal +import sys +import time +import webbrowser +import pyautogui + +from langchain.tools import BaseTool +from pydantic import BaseModel, Field, create_model + +from agent.llm import Qwen36_35B_A3B_FP8_LLM +from utils.env_utils import BROWSER_MAP + + +class Websearchclass(BaseTool): + + args_schema : type[BaseModel] = create_model("Input_type", query=(str, Field(description="搜索查询内容"))) + name : str = "web_search_class" + description : str = "一个网络搜索工具，可以根据用户的查询返回相关的搜索结果。仅当需要查询近期知识或已有知识无法解答需要联网查询时调用，其余任何情况严禁调用。" + + def _get_search_wrapper(self): from langchain_community.utilities import GoogleSerperAPIWrapper - # 这里直接使用了一个示例的 Serper API 密钥，实际使用时请替换为你自己的密钥 SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5" - # LangChain 官方封装的 Serper API 工具 - search = GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048) - # 直接调用 run 方法获取结果，LangChain 已处理好格式 - return search.run(query) - except Exception as e: - return f"执行网络搜索时出错: {str(e)}" - -@tool("date_query") -def date_query() -> str: - """ - 一个用于执行日期查询的工具。它接收一个查询语句作为输入，并返回相应的日期结果。 - - Returns: - str: 格式化后的日期查询结果 - """ - try: - # 显示当前日期和时间 - from datetime import datetime - now = datetime.now() - return now.strftime("%Y-%m-%d %H:%M:%S") + return GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048) - except Exception as e: - return f"执行日期查询时出错: {str(e)}" + def _run(self, query: str) -> str: + try: + return self._get_search_wrapper().run(query) + except Exception as e: + return f"执行网络搜索时出错: {str(e)}" + + async def _arun(self, query: str) -> str: + try: + return await self._get_search_wrapper().arun(query) + except Exception as e: + return f"执行网络搜索时出错: {str(e)}" + +class DateQueryClass(BaseTool): + + # 定义工具的输入参数结构，这里没有输入参数，所以是一个空的模型 + args_schema : type[BaseModel] = create_model("NoneInput") + name : str = "date_query_class" + description : str = "一个日期查询工具，可以返回当前的日期信息。仅当需要查询当前时间时调用，其余任何情况严禁调用。" + + def _run(self) -> str: + try: + from datetime import datetime + now = datetime.now() + return now.strftime("%Y-%m-%d %H:%M:%S") + except Exception as e: + return f"执行日期查询时出错: {str(e)}" + + async def _arun(self) -> str: + return self._run() + + +class ScreenCaptureDescribeClass(BaseTool): + """截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。""" + + args_schema: type[BaseModel] = create_model( + "ScreenCaptureInput", + focus_query=( + Optional[str], + Field( + default=None, + description="可选。希望重点观察的界面或问题，例如「错误提示全文」「浏览器地址栏」；留空则整体描述屏幕。", + ), + ), + ) + name: str = "screen_capture_describe" + description: str = ( + "截取当前主屏幕画面，并用视觉模型转为文字描述。" + "注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。" + ) + + def _run(self, focus_query: Optional[str] = None) -> str: + # 1. 确保 pyautogui 可用 + try: + import pyautogui + except ImportError as e: + return ( + f"未安装 pyautogui，无法截屏: {e}。" + "请在项目环境中执行: pip install pyautogui" + ) + + # 2. 截屏并转为 PNG 字节流 + try: + pil_img = pyautogui.screenshot() + buf = io.BytesIO() + pil_img.save(buf, format="PNG") + raw_png = buf.getvalue() + except Exception as e: + return f"截屏失败: {e}" + + # 3. 构建描述指令 + query = (focus_query or "").strip() + + # 4. 调用与对话图片上传完全相同的视觉模型进行描述 + try: + # Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义 + text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query) + except Exception as e: + return f"屏幕图像视觉解析失败: {e}" + + if not (text or "").strip(): + return "（已截屏，但模型未返回有效描述。）" + return f"【当前屏幕视觉描述】\n{text.strip()}" + +class BrowserControlClass(BaseTool): + """统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL""" + + args_schema: type[BaseModel] = create_model( + "BrowserControlInput", + action=(Literal["open", "close"], Field( + default="open", + description="操作类型：'open' 打开/刷新，'close' 关闭" + )), + url=(Optional[str], Field( + default=None, + description="URL 地址、'current' 表示当前页，留空则操作整个浏览器窗口" + )), + browser=(Optional[str], Field( + default=None, + description="浏览器名称，如 'edge', 'chrome', 'firefox'，不指定则使用系统默认" + )), + ) + name: str = "browser_control" + description: str = ( + "统一控制浏览器：打开/关闭浏览器、刷新/关闭标签页、打开指定 URL。\n" + "- action='open', url=None: 打开浏览器首页\n" + "- action='open', url='current': 刷新当前标签页\n" + "- action='open', url='https://...': 在新标签页打开指定 URL\n" + "- action='close', url=None: 关闭整个浏览器窗口\n" + "- action='close', url='current': 关闭当前标签页\n" + "- action='close', url='https://...': 关闭当前活动标签页" + "仅用于进行浏览器操作时调用，其余任何情况严禁调用。" + ) + + # ---------- 辅助方法 ---------- + def _get_browser_info(self, browser_name: Optional[str]): + """获取浏览器控制器和窗口标题关键字""" + if browser_name is None: + return webbrowser.get(), None + + name_lower = browser_name.lower().strip() + if name_lower not in BROWSER_MAP: + raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}") + + reg_name, title_keyword = BROWSER_MAP[name_lower] + try: + controller = webbrowser.get(reg_name) + except webbrowser.Error: + controller = webbrowser.get(browser_name) + return controller, title_keyword + + def _activate_window(self, title_keyword: str) -> bool: + """激活浏览器窗口（仅 Windows）""" + if sys.platform != "win32": + return False + try: + time.sleep(0.3) + windows = pyautogui.getWindowsWithTitle(title_keyword) + if windows: + win = windows[0] + if win.isMinimized: + win.restore() + win.activate() + return True + except Exception: + pass + return False + + def _send_keys(self, *keys): + """发送快捷键，根据平台调整""" + pyautogui.hotkey(*keys) + + # ---------- 核心操作 ---------- + def _open_action(self, url: Optional[str], browser: Optional[str]): + """执行打开类操作""" + controller, title_keyword = self._get_browser_info(browser) + + if url is None: + # 仅打开浏览器首页（激活或启动） + controller.open("about:blank") + if title_keyword: + self._activate_window(title_keyword) + browser_info = f" (使用 {browser})" if browser else "" + return f"已打开浏览器首页{browser_info}" + + elif url.lower() == "current": + # 刷新当前页 + if title_keyword: + if not self._activate_window(title_keyword): + return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新" + time.sleep(0.2) + if sys.platform == "darwin": + self._send_keys("command", "r") + else: + self._send_keys("f5") + return "已刷新当前标签页" + + else: + # 打开指定 URL + controller.open(url) + if title_keyword: + self._activate_window(title_keyword) + return f"已尝试打开 URL: {url}" + + def _close_action(self, url: Optional[str], browser: Optional[str]): + """执行关闭类操作""" + _, title_keyword = self._get_browser_info(browser) + + if url is None: + # 关闭整个浏览器 + if title_keyword: + if not self._activate_window(title_keyword): + return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭" + time.sleep(0.2) + if sys.platform == "darwin": + self._send_keys("command", "q") + else: + self._send_keys("alt", "f4") + return "已尝试关闭浏览器窗口" + + elif url.lower() == "current": + # 关闭当前标签页 + if title_keyword: + if not self._activate_window(title_keyword): + return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页" + time.sleep(0.2) + if sys.platform == "darwin": + self._send_keys("command", "w") + else: + self._send_keys("ctrl", "w") + return "已尝试关闭当前标签页" + + def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str: + try: + if action == "open": + return self._open_action(url, browser) + elif action == "close": + return self._close_action(url, browser) + else: + return f"不支持的操作类型: {action}" + except Exception as e: + return f"执行浏览器操作时出错: {e}" if __name__ == "__main__": - # print(web_search.name) # 输出工具名称 - # print(web_search.description) # 输出工具描述 - # print(web_search.args_schema.model_json_schema()) # 输出工具参数结构 - # # 测试工具 - # query = "Python 中的列表和元组有什么区别？" - # result = web_search.invoke({"query": query}) - # print(result) - print(date_query.name) # 输出工具名称 - print(date_query.description) # 输出工具描述 - print(date_query.args_schema.model_json_schema()) # 输出工具参数结构 - print(date_query.invoke({})) # 测试工具 - + # 这里可以进行一些简单的测试，例如创建工具实例并调用方法 + pass diff --git a/src/agent/tools/tool_main.py b/src/agent/tools/tool_main.py deleted file mode 100644 index ff2e9f5..0000000 --- a/src/agent/tools/tool_main.py +++ /dev/null @@ -1,252 +0,0 @@ -from __future__ import annotations - -import io -from typing import Optional, Literal -import sys -import time -import webbrowser -import pyautogui - -from langchain.tools import BaseTool -from pydantic import BaseModel, Field, create_model - -from agent.llm import Qwen36_35B_A3B_FP8_LLM -from utils.env_utils import BROWSER_MAP - - -class Websearchclass(BaseTool): - - args_schema : type[BaseModel] = create_model("Input_type", query=(str, Field(description="搜索查询内容"))) - name : str = "web_search_class" - description : str = "一个网络搜索工具，可以根据用户的查询返回相关的搜索结果。仅当需要查询近期知识或已有知识无法解答需要联网查询时调用，其余任何情况严禁调用。" - - def _get_search_wrapper(self): - from langchain_community.utilities import GoogleSerperAPIWrapper - SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5" - return GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048) - - def _run(self, query: str) -> str: - try: - return self._get_search_wrapper().run(query) - except Exception as e: - return f"执行网络搜索时出错: {str(e)}" - - async def _arun(self, query: str) -> str: - try: - return await self._get_search_wrapper().arun(query) - except Exception as e: - return f"执行网络搜索时出错: {str(e)}" - -class DateQueryClass(BaseTool): - - # 定义工具的输入参数结构，这里没有输入参数，所以是一个空的模型 - args_schema : type[BaseModel] = create_model("NoneInput") - name : str = "date_query_class" - description : str = "一个日期查询工具，可以返回当前的日期信息。仅当需要查询当前时间时调用，其余任何情况严禁调用。" - - def _run(self) -> str: - try: - from datetime import datetime - now = datetime.now() - return now.strftime("%Y-%m-%d %H:%M:%S") - except Exception as e: - return f"执行日期查询时出错: {str(e)}" - - async def _arun(self) -> str: - return self._run() - - -class ScreenCaptureDescribeClass(BaseTool): - """截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。""" - - args_schema: type[BaseModel] = create_model( - "ScreenCaptureInput", - focus_query=( - Optional[str], - Field( - default=None, - description="可选。希望重点观察的界面或问题，例如「错误提示全文」「浏览器地址栏」；留空则整体描述屏幕。", - ), - ), - ) - name: str = "screen_capture_describe" - description: str = ( - "截取当前主屏幕画面，并用视觉模型转为文字描述。" - "注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。" - ) - - def _run(self, focus_query: Optional[str] = None) -> str: - # 1. 确保 pyautogui 可用 - try: - import pyautogui - except ImportError as e: - return ( - f"未安装 pyautogui，无法截屏: {e}。" - "请在项目环境中执行: pip install pyautogui" - ) - - # 2. 截屏并转为 PNG 字节流 - try: - pil_img = pyautogui.screenshot() - buf = io.BytesIO() - pil_img.save(buf, format="PNG") - raw_png = buf.getvalue() - except Exception as e: - return f"截屏失败: {e}" - - # 3. 构建描述指令 - query = (focus_query or "").strip() - - # 4. 调用与对话图片上传完全相同的视觉模型进行描述 - try: - # Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义 - text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query) - except Exception as e: - return f"屏幕图像视觉解析失败: {e}" - - if not (text or "").strip(): - return "（已截屏，但模型未返回有效描述。）" - return f"【当前屏幕视觉描述】\n{text.strip()}" - -class BrowserControlClass(BaseTool): - """统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL""" - - args_schema: type[BaseModel] = create_model( - "BrowserControlInput", - action=(Literal["open", "close"], Field( - default="open", - description="操作类型：'open' 打开/刷新，'close' 关闭" - )), - url=(Optional[str], Field( - default=None, - description="URL 地址、'current' 表示当前页，留空则操作整个浏览器窗口" - )), - browser=(Optional[str], Field( - default=None, - description="浏览器名称，如 'edge', 'chrome', 'firefox'，不指定则使用系统默认" - )), - ) - name: str = "browser_control" - description: str = ( - "统一控制浏览器：打开/关闭浏览器、刷新/关闭标签页、打开指定 URL。\n" - "- action='open', url=None: 打开浏览器首页\n" - "- action='open', url='current': 刷新当前标签页\n" - "- action='open', url='https://...': 在新标签页打开指定 URL\n" - "- action='close', url=None: 关闭整个浏览器窗口\n" - "- action='close', url='current': 关闭当前标签页\n" - "- action='close', url='https://...': 关闭当前活动标签页" - "仅用于进行浏览器操作时调用，其余任何情况严禁调用。" - ) - - # ---------- 辅助方法 ---------- - def _get_browser_info(self, browser_name: Optional[str]): - """获取浏览器控制器和窗口标题关键字""" - if browser_name is None: - return webbrowser.get(), None - - name_lower = browser_name.lower().strip() - if name_lower not in BROWSER_MAP: - raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}") - - reg_name, title_keyword = BROWSER_MAP[name_lower] - try: - controller = webbrowser.get(reg_name) - except webbrowser.Error: - controller = webbrowser.get(browser_name) - return controller, title_keyword - - def _activate_window(self, title_keyword: str) -> bool: - """激活浏览器窗口（仅 Windows）""" - if sys.platform != "win32": - return False - try: - time.sleep(0.3) - windows = pyautogui.getWindowsWithTitle(title_keyword) - if windows: - win = windows[0] - if win.isMinimized: - win.restore() - win.activate() - return True - except Exception: - pass - return False - - def _send_keys(self, *keys): - """发送快捷键，根据平台调整""" - pyautogui.hotkey(*keys) - - # ---------- 核心操作 ---------- - def _open_action(self, url: Optional[str], browser: Optional[str]): - """执行打开类操作""" - controller, title_keyword = self._get_browser_info(browser) - - if url is None: - # 仅打开浏览器首页（激活或启动） - controller.open("about:blank") - if title_keyword: - self._activate_window(title_keyword) - browser_info = f" (使用 {browser})" if browser else "" - return f"已打开浏览器首页{browser_info}" - - elif url.lower() == "current": - # 刷新当前页 - if title_keyword: - if not self._activate_window(title_keyword): - return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新" - time.sleep(0.2) - if sys.platform == "darwin": - self._send_keys("command", "r") - else: - self._send_keys("f5") - return "已刷新当前标签页" - - else: - # 打开指定 URL - controller.open(url) - if title_keyword: - self._activate_window(title_keyword) - return f"已尝试打开 URL: {url}" - - def _close_action(self, url: Optional[str], browser: Optional[str]): - """执行关闭类操作""" - _, title_keyword = self._get_browser_info(browser) - - if url is None: - # 关闭整个浏览器 - if title_keyword: - if not self._activate_window(title_keyword): - return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭" - time.sleep(0.2) - if sys.platform == "darwin": - self._send_keys("command", "q") - else: - self._send_keys("alt", "f4") - return "已尝试关闭浏览器窗口" - - elif url.lower() == "current": - # 关闭当前标签页 - if title_keyword: - if not self._activate_window(title_keyword): - return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页" - time.sleep(0.2) - if sys.platform == "darwin": - self._send_keys("command", "w") - else: - self._send_keys("ctrl", "w") - return "已尝试关闭当前标签页" - - def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str: - try: - if action == "open": - return self._open_action(url, browser) - elif action == "close": - return self._close_action(url, browser) - else: - return f"不支持的操作类型: {action}" - except Exception as e: - return f"执行浏览器操作时出错: {e}" - -if __name__ == "__main__": - # 这里可以进行一些简单的测试，例如创建工具实例并调用方法 - pass diff --git a/src/agent/tools/tool_rag.py b/src/agent/tools/tool_rag.py new file mode 100644 index 0000000..affac4a --- /dev/null +++ b/src/agent/tools/tool_rag.py @@ -0,0 +1,373 @@ +# agent/tools/tool_rag.py +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import time +from datetime import datetime +from typing import List, Optional, Union, Literal + +import chromadb +import requests +from langchain.tools import BaseTool +from pydantic import BaseModel, Field, create_model +from langchain.text_splitter import RecursiveCharacterTextSplitter + +from utils.env_utils import RAG_BASE_URL, RAG_CHUNK_SIZE, RAG_CHUNK_OVERLAP + +# ---------- 新增 RAG 工具 ---------- +# Chroma 持久化路径：agent/memory/chroma_rag_db +CHROMA_PERSIST_DIR = os.path.join(os.path.dirname(__file__), '..', 'memory', 'chroma_rag_db') + +# 全局 Chroma 客户端和集合缓存 +_chroma_client = None +_collection_cache = {} + +def get_chroma_client(): + global _chroma_client + if _chroma_client is None: + _chroma_client = chromadb.PersistentClient(path=CHROMA_PERSIST_DIR) + return _chroma_client + +def get_collection(agent_name: str): + """获取或创建某个 agent 的 Collection""" + client = get_chroma_client() + collection_name = f"rag_{agent_name}" + if collection_name not in _collection_cache: + # 若 Collection 不存在则创建，使用余弦相似度 + try: + collection = client.get_collection(name=collection_name, embedding_function=None) + except Exception: + collection = client.create_collection( + name=collection_name, + metadata={"hnsw:space": "cosine"} + ) + _collection_cache[collection_name] = collection + return _collection_cache[collection_name] + +# ---------- Embedding 接口封装 ---------- +def embed_texts(queries: List[str], documents: List[str] = None) -> tuple: + """ + 调用 /embed，返回 (query_embeddings, doc_embeddings) + """ + payload = { + "task_description": "Given a web search query, retrieve relevant passages that answer the query", + "queries": queries, + "documents": documents or [] + } + try: + resp = requests.post(f"{RAG_BASE_URL}/embed", json=payload, timeout=30) + resp.raise_for_status() + data = resp.json() + return data["query_embeddings"], data.get("doc_embeddings", []) + except Exception as e: + raise RuntimeError(f"Embedding 请求失败: {e}") + +def rerank_documents(query: str, documents: List[str]) -> List[float]: + """调用 /rerank，返回排序后的分数列表""" + payload = { + "task_description": "Given a web search query, retrieve relevant passages that answer the query", + "query": query, + "documents": documents + } + try: + resp = requests.post(f"{RAG_BASE_URL}/rerank", json=payload, timeout=30) + resp.raise_for_status() + return resp.json()["scores"] + except Exception as e: + raise RuntimeError(f"Reranker 请求失败: {e}") + +# ---------- 文本分块 ---------- +def split_into_chunks(text: str) -> List[str]: + splitter = RecursiveCharacterTextSplitter( + chunk_size=CHUNK_SIZE, + chunk_overlap=CHUNK_OVERLAP, + separators=["\n\n", "\n", " ", ""] + ) + return splitter.split_text(text) + +# ---------- 文件解析 ---------- +def parse_file_to_chunks(file_path: str) -> List[dict]: + """ + 解析文件，返回每个 chunk 的字典，包含： + - content: 文本内容 + - source: 详细来源描述 + """ + ext = os.path.splitext(file_path)[1].lower() + chunks = [] + + if ext == ".txt" or ext == ".md": + with open(file_path, "r", encoding="utf-8") as f: + text = f.read() + texts = split_into_chunks(text) + for i, t in enumerate(texts): + chunks.append({ + "content": t, + "source": f"{os.path.basename(file_path)} (chunk {i+1}/{len(texts)})" + }) + + elif ext == ".json": + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + text = json.dumps(data, ensure_ascii=False, indent=2) + texts = split_into_chunks(text) + for i, t in enumerate(texts): + chunks.append({ + "content": t, + "source": f"{os.path.basename(file_path)} (JSON, chunk {i+1}/{len(texts)})" + }) + + elif ext == ".csv": + import pandas as pd + df = pd.read_csv(file_path) + text = df.to_string(index=False) + texts = split_into_chunks(text) + for i, t in enumerate(texts): + chunks.append({ + "content": t, + "source": f"{os.path.basename(file_path)} (CSV, chunk {i+1}/{len(texts)})" + }) + + elif ext == ".docx": + from docx import Document + doc = Document(file_path) + full_text = "\n".join([p.text for p in doc.paragraphs]) + texts = split_into_chunks(full_text) + for i, t in enumerate(texts): + chunks.append({ + "content": t, + "source": f"{os.path.basename(file_path)} (DOCX, chunk {i+1}/{len(texts)})" + }) + + elif ext == ".pdf": + import pdfplumber + with pdfplumber.open(file_path) as pdf: + for page_num, page in enumerate(pdf.pages, start=1): + page_text = page.extract_text() or "" + if page_text.strip(): + page_chunks = split_into_chunks(page_text) + for i, t in enumerate(page_chunks): + chunks.append({ + "content": t, + "source": f"{os.path.basename(file_path)} (Page {page_num}, chunk {i+1})" + }) + + else: + raise ValueError(f"不支持的文件格式: {ext}") + + return chunks + +# ---------- RAG 工具本体 ---------- +class RAGTool(BaseTool): + name: str = "rag_tool" + description: str = ( + "通用知识库工具，支持增删查操作。\n" + "参数说明：\n" + "- agent_name: 所属 Agent 名称或名称列表，必填。\n" + "- action_type: 操作类型，必填。可选：'query'(查询), 'add_document'(添加文档), 'clear'(清除)。\n" + "- content: 查询内容(用于query) 或 直接文本内容(用于add_document，与source二选一)。\n" + "- source: 文件路径(用于add_document，与content二选一)。\n" + "- filter: 过滤条件(用于query或clear)。\n" + "使用示例：\n" + "- 查询：action_type='query', content='用户问题', agent_name='main'\n" + "- 添加文本：action_type='add_document', content='知识文本', agent_name='main'\n" + "- 添加文件：action_type='add_document', source='/path/to/file.pdf', agent_name=['agent1','agent2']\n" + "- 清除：action_type='clear', filter='某个来源', agent_name='main'" + ) + + args_schema: type[BaseModel] = create_model( + "RAGInput", + agent_name=(Union[str, List[str]], Field(..., description="所属 Agent 名称或名称列表")), + action_type=(Literal["query", "add_document", "clear"], Field(..., description="操作类型")), + content=(Optional[str], Field(None, description="查询内容或直接添加的文本内容")), + source=(Optional[str], Field(None, description="文件路径，用于添加文档")), + filter=(Optional[Union[str, List[str]]], Field(None, description="过滤条件，用于查询或清除")) + ) + + def _run( + self, + agent_name: Union[str, List[str]], + action_type: str, + content: Optional[str] = None, + source: Optional[str] = None, + filter: Optional[Union[str, List[str]]] = None + ) -> str: + try: + # 标准化 agent_name 为列表 + agents = [agent_name] if isinstance(agent_name, str) else agent_name + + if action_type == "query": + return self._handle_query(agents, content, filter) + elif action_type == "add_document": + return self._handle_add(agents, content, source) + elif action_type == "clear": + return self._handle_clear(agents, filter) + else: + return f"未知操作: {action_type}" + except Exception as e: + return f"RAG 工具执行出错: {str(e)}" + + # ---------- 查询 ---------- + def _handle_query(self, agents: List[str], query: str, filter: Optional[Union[str, List[str]]]) -> str: + if not query: + return "查询内容不能为空。" + + # 1. 获取查询向量 + q_embs, _ = embed_texts(queries=[query]) + query_vector = q_embs[0] + + # 2. 从相关 Collection 中检索 + all_results = [] + for agent in agents: + coll = get_collection(agent) + # 构建 where 条件（基础过滤可叠加元数据过滤，这里简单处理） + where = None + if filter: + if isinstance(filter, str): + # 简单实现：按 source 字段包含 filter 字符串 + where = {"source": {"$contains": filter}} + else: + # 列表情况，暂不支持，返回空 + pass + + res = coll.query( + query_embeddings=[query_vector], + n_results=10, + where=where + ) + # 整理结果：[(id, document, metadata), ...] + if res["ids"] and res["ids"][0]: + ids = res["ids"][0] + docs = res["documents"][0] + metas = res["metadatas"][0] + for i, d, m in zip(ids, docs, metas): + all_results.append({"id": i, "document": d, "metadata": m}) + + if not all_results: + return "未在知识库中找到相关内容。" + + # 去重（同一个文档可能被多个 agent 同时索引） + seen = set() + unique_results = [] + for r in all_results: + if r["document"] not in seen: + seen.add(r["document"]) + unique_results.append(r) + unique_results = unique_results[:10] # 确保最多10条 + + # 3. Reranker 精排 + docs_for_rerank = [r["document"] for r in unique_results] + scores = rerank_documents(query, docs_for_rerank) + + # 按分数降序取前3 + sorted_items = sorted( + zip(unique_results, scores), + key=lambda x: x[1], + reverse=True + )[:3] + + if not sorted_items: + return "Reranker 未返回有效结果。" + + # 拼接输出 + outputs = [] + for item, score in sorted_items: + meta = item["metadata"] + source_info = meta.get("source", "未知来源") + outputs.append(f"【来源：{source_info}（相关度 {score:.3f}）】\n{item['document']}") + + return "\n\n".join(outputs) + + # ---------- 添加文档 ---------- + def _handle_add(self, agents: List[str], content: Optional[str], source: Optional[str]) -> str: + if not content and not source: + return "必须提供 content 或 source。" + if content and source: + return "content 和 source 只能提供一个。" + + # 1. 准备 chunks 列表 + chunks_meta = [] # 元素为 (text, source_desc) + timestamp = datetime.now().isoformat() + + if content: + # 直接使用 content 分块 + texts = split_into_chunks(content) + chunks_meta = [(t, "直接输入") for t in texts] + + elif source: + # 解析文件 + if not os.path.isfile(source): + return f"文件不存在: {source}" + chunks_meta = [(c["content"], c["source"]) for c in parse_file_to_chunks(source)] + + if not chunks_meta: + return "文档内容为空，未添加任何条目。" + + # 2. 批量获取文档向量 + doc_texts = [cm[0] for cm in chunks_meta] + _, doc_embs = embed_texts(queries=["dummy"], documents=doc_texts) + + # 3. 为每个 agent 写入 + total_added = 0 + for agent in agents: + coll = get_collection(agent) + ids = [] + embeddings = [] + metadatas = [] + documents = [] + + for idx, (text, src_desc) in enumerate(chunks_meta): + # 使用 hash 作为唯一 ID + uid = hashlib.md5(f"{agent}_{timestamp}_{text[:80]}".encode()).hexdigest() + ids.append(uid) + embeddings.append(doc_embs[idx]) + metadatas.append({ + "source": src_desc, + "agent_name": agent, + "timestamp": timestamp, + }) + documents.append(text) + + coll.add( + ids=ids, + embeddings=embeddings, + metadatas=metadatas, + documents=documents + ) + total_added += len(ids) + + return f"成功添加 {total_added} 条知识块到 {', '.join(agents)}。" + + # ---------- 清除 ---------- + def _handle_clear(self, agents: List[str], filter: Optional[Union[str, List[str]]]) -> str: + if not filter: + return "清除操作必须提供 filter 条件。" + + total_deleted = 0 + for agent in agents: + coll = get_collection(agent) + # 根据 filter 构建 where，当前简化：假设 filter 是 source 的包含条件 + where = None + if isinstance(filter, str): + where = {"source": {"$contains": filter}} + elif isinstance(filter, list): + # 多个条件用 $or + where = {"$or": [{"source": {"$contains": f}} for f in filter]} + + try: + # 查询要删除的 IDs + res = coll.get(where=where) + if res["ids"]: + coll.delete(ids=res["ids"]) + total_deleted += len(res["ids"]) + except Exception as e: + return f"清除时出错 (agent={agent}): {e}" + + return f"已从 {', '.join(agents)} 中删除 {total_deleted} 条记录。" + + async def _arun(self, **kwargs): + # 直接调用同步版本 + return self._run(**kwargs) + \ No newline at end of file diff --git a/src/agent/utils/__pycache__/__init__.cpython-312.pyc b/src/agent/utils/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index c72cf9f..0000000 Binary files a/src/agent/utils/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc b/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc deleted file mode 100644 index 37b3922..0000000 Binary files a/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc and /dev/null differ diff --git a/src/agent/utils/__pycache__/env_utils.cpython-312.pyc b/src/agent/utils/__pycache__/env_utils.cpython-312.pyc deleted file mode 100644 index e8a9f47..0000000 Binary files a/src/agent/utils/__pycache__/env_utils.cpython-312.pyc and /dev/null differ diff --git a/src/agent/utils/__pycache__/image_utils.cpython-312.pyc b/src/agent/utils/__pycache__/image_utils.cpython-312.pyc deleted file mode 100644 index 8894438..0000000 Binary files a/src/agent/utils/__pycache__/image_utils.cpython-312.pyc and /dev/null differ diff --git a/src/agent/utils/agent_utils.py b/src/agent/utils/agent_utils.py index f8e77ab..a449fd7 100644 --- a/src/agent/utils/agent_utils.py +++ b/src/agent/utils/agent_utils.py @@ -1,15 +1,23 @@ -# agent/agent_utils.py +# agent/utils/agent_utils.py import json import os import shutil +import sqlite3 +from typing import List, Tuple, Any from langchain_community.chat_message_histories import SQLChatMessageHistory -from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_core.messages import BaseMessage, SystemMessage, HumanMessage +from langchain_core.prompts import ChatPromptTemplate from agent.memory.system_prompt import SUMMARY_SYSTEM_PROMPT +from agent.utils.env_utils import ( + LLM_CONTEXT_WINDOW, + MINI_COMPRESS_RATE, + HARD_COMPRESS_RATE, + CURRENT_TURNS, +) -# 关键改动：__file__ 在 agent/utils/agent_utils.py 下， -# 取两次 dirname 得到 agent 目录（即 src/agent） +# ---------- 目录与数据库路径 ---------- _AGENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) HISTORY_ROOT = os.path.join(_AGENT_DIR, "history") _DB_DIR = os.path.join(HISTORY_ROOT, "data_base") @@ -18,58 +26,234 @@ MULTI_INPUTS_ROOT = os.path.join(HISTORY_ROOT, "multi_inputs") def ensure_dirs_and_migrate_db() -> None: os.makedirs(_DB_DIR, exist_ok=True) - os.makedirs(os.path.join(HISTORY_ROOT, "multi_inputs"), exist_ok=True) + os.makedirs(MULTI_INPUTS_ROOT, exist_ok=True) + # 迁移旧数据库 old_db = os.path.join(HISTORY_ROOT, "chat_history.db") if os.path.isfile(old_db) and not os.path.isfile(CHAT_HISTORY_DB_FS): shutil.move(old_db, CHAT_HISTORY_DB_FS) + conn = sqlite3.connect(CHAT_HISTORY_DB_FS) + # 创建 message_store 表（与 SQLChatMessageHistory 一致） + conn.execute(""" + CREATE TABLE IF NOT EXISTS message_store ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + message TEXT NOT NULL + ) + """) + # 创建 session_meta 表（我们的元数据表） + conn.execute(""" + CREATE TABLE IF NOT EXISTS session_meta ( + session_id TEXT NOT NULL, + agent_name TEXT NOT NULL, + cursor_index INTEGER DEFAULT -1, + compressed_content TEXT DEFAULT '', + last_total_tokens INTEGER DEFAULT 0, + PRIMARY KEY (session_id, agent_name) + ) + """) + conn.commit() + conn.close() + ensure_dirs_and_migrate_db() +# ---------- 历史记录操作 ---------- def get_history(session_id: str) -> SQLChatMessageHistory: return SQLChatMessageHistory( session_id=session_id, connection="sqlite:///" + CHAT_HISTORY_DB_FS.replace("\\", "/"), ) -def summarize_history(query, config): - """ - 剪辑最近8条消息，并对更早的消息生成摘要。 - 返回 {'summary': str|None, 'chat_history': list[BaseMessage]} - """ - session_id = config["configurable"]["session_id"] - if not session_id: - raise ValueError("Session ID is required in the config.") - chat_history = get_history(session_id) - stored_messages = chat_history.messages - - if len(stored_messages) <= 8: - return {"summary": None, "chat_history": stored_messages} - - recent = stored_messages[-8:] - old = stored_messages[:-8] - - summary_prompt = ChatPromptTemplate.from_messages([ - ("system", SUMMARY_SYSTEM_PROMPT), - MessagesPlaceholder(variable_name="messages_to_summarize", optional=True), - ("human", "请生成包含上述对话核心内容的摘要，保留重要事实和决策。"), - ]) - - llm = config.get("llm") - chain = summary_prompt | llm - summary = chain.invoke( - {"messages_to_summarize": old}, - config={ - "metadata": { - "output_type": "text", - "audio_output_dir": None, - } - } +# ---------- session_meta 元数据管理 ---------- +def get_session_meta(session_id: str, agent_name: str) -> dict: + conn = sqlite3.connect(CHAT_HISTORY_DB_FS) + cur = conn.execute( + "SELECT cursor_index, compressed_content, last_total_tokens FROM session_meta WHERE session_id=? AND agent_name=?", + (session_id, agent_name) ) - return {"summary": summary.content, "chat_history": recent} + row = cur.fetchone() + conn.close() + if row: + return { + "cursor_index": row[0], + "compressed_content": row[1], + "last_total_tokens": row[2] + } + return {"cursor_index": -1, "compressed_content": "", "last_total_tokens": 0} +def update_session_meta(session_id: str, agent_name: str, + cursor_index: int, compressed_content: str, last_total_tokens: int) -> None: + conn = sqlite3.connect(CHAT_HISTORY_DB_FS) + conn.execute(""" + INSERT INTO session_meta (session_id, agent_name, cursor_index, compressed_content, last_total_tokens) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(session_id, agent_name) DO UPDATE SET + cursor_index=excluded.cursor_index, + compressed_content=excluded.compressed_content, + last_total_tokens=excluded.last_total_tokens + """, (session_id, agent_name, cursor_index, compressed_content, last_total_tokens)) + conn.commit() + conn.close() +# ---------- 工具函数 ---------- +def _get_messages_since_cursor(history: SQLChatMessageHistory, cursor_index: int) -> List[BaseMessage]: + all_msgs = history.messages + if cursor_index < 0 or cursor_index >= len(all_msgs) - 1: + return all_msgs + return all_msgs[cursor_index + 1:] + +def _count_turns(messages: List[BaseMessage]) -> int: + turns = 0 + i = 0 + n = len(messages) + while i < n: + if messages[i].type == "human": + if i + 1 < n and messages[i + 1].type in ("ai", "assistant"): + turns += 1 + i += 2 + else: + i += 1 + else: + i += 1 + return turns + +def _last_turn_end_index(messages: List[BaseMessage]) -> int: + for i in range(len(messages) - 1, -1, -1): + if messages[i].type in ("ai", "assistant"): + return i + return -1 + +# ---------- 压缩函数 ---------- +def _lightweight_compress(llm, messages_to_compress: List[BaseMessage]) -> str: + """轻量压缩：保留关键事实、实体、决策，删除冗余""" + prompt = ChatPromptTemplate.from_messages([ + ("system", SUMMARY_SYSTEM_PROMPT), + ("human", "请对以下对话进行轻量压缩，保留核心事实、实体、决策和对话流向。\n\n{chat}"), + ]) + chain = prompt | llm + chat_text = "\n".join([ + f"{'用户' if m.type == 'human' else '助手'}: {m.content}" for m in messages_to_compress + ]) + print("轻量化压缩函数被调用") + print("需要压缩的内容为"+chat_text) + result = chain.invoke({"chat": chat_text}, config={"metadata": {"output_type": "text","audio_output_dir": None,}}) + return result.content + +def _hard_compress(llm, compressed_existing: str, new_chat_text: str) -> str: + """全量压缩：将已有压缩摘要和新对话内容融合为最终摘要""" + prompt = ChatPromptTemplate.from_messages([ + ("system", SUMMARY_SYSTEM_PROMPT), + ("human", "这是之前的对话摘要：\n{previous_summary}\n\n以及后续对话内容：\n{new_chat}\n\n请生成包含所有信息的最终摘要，保留重要事实和决策。"), + ]) + chain = prompt | llm + print("全量压缩函数被调用") + result = chain.invoke({ + "previous_summary": compressed_existing, + "new_chat": new_chat_text + }, config={"metadata": {"output_type": "text"}}) + return result.content + +# ---------- 核心：上下文感知构建器 (summarizer) ---------- +def summarizer( + session_id: str, + agent_name: str, + system_prompt: str, + user_content: str, + llm: Any +) -> Tuple[List[BaseMessage], bool]: + """ + 根据上下文窗口与上一轮 token 使用量，动态压缩历史消息并返回完整消息列表。 + :param session_id: 会话ID + :param agent_name: Agent 名称 + :param system_prompt: 系统提示词 + :param user_content: 当前用户输入文本（纯文本或已处理的多模态文本） + :param llm: 语言模型实例（用于压缩） + :return: (messages, context_exceeded) + messages: 构建好的消息列表 + context_exceeded: 是否超过 90% 窗口，需要提示新建会话 + """ + meta = get_session_meta(session_id, agent_name) + cursor_index = meta["cursor_index"] + compressed_content = meta["compressed_content"] + last_total_tokens = meta["last_total_tokens"] + + # 检查上下文是否已满 + if last_total_tokens > LLM_CONTEXT_WINDOW * 0.9: + return [], True + + # 判断是否需要触发压缩 + need_light = False + need_hard = False + if last_total_tokens > 0: + if last_total_tokens > LLM_CONTEXT_WINDOW * HARD_COMPRESS_RATE: + need_hard = True + elif last_total_tokens > LLM_CONTEXT_WINDOW * MINI_COMPRESS_RATE: + need_light = True + + history = get_history(session_id) + all_messages = history.messages + + if need_light: + # 轻量压缩：保留最近 CURRENT_TURNS 轮对话不压缩 + turns = _count_turns(all_messages) + keep_turns = min(CURRENT_TURNS, turns) + # 找到需要保留的轮数之后的最后一条消息索引 + idx = len(all_messages) - 1 + turn_count = 0 + while idx >= 0: + if all_messages[idx].type in ("ai", "assistant"): + turn_count += 1 + if turn_count == keep_turns: + break + idx -= 1 + new_cursor = idx - 1 if idx >= 0 else -1 # idx 是保留轮次开始的前一条消息索引 + + # 待压缩内容 = 游标及之前的所有消息 + to_compress = [] + if new_cursor >= 0: + to_compress = all_messages[:new_cursor + 1] + if compressed_content: + to_compress.insert(0, HumanMessage(content=f"[先前摘要] {compressed_content}")) + + new_compressed = _lightweight_compress(llm, to_compress) if to_compress else compressed_content + # 更新游标和压缩内容 + update_session_meta(session_id, agent_name, new_cursor, new_compressed, last_total_tokens) + meta["cursor_index"] = new_cursor + meta["compressed_content"] = new_compressed + + elif need_hard: + # 全量压缩：将已有压缩内容 + 游标后的所有消息合并为最终摘要 + after_cursor = _get_messages_since_cursor(history, cursor_index) + new_chat_text = "\n".join( + [f"{'用户' if m.type == 'human' else '助手'}: {m.content}" for m in after_cursor] + ) + if compressed_content: + new_compressed = _hard_compress(llm, compressed_content, new_chat_text) + else: + new_compressed = _hard_compress(llm, "", new_chat_text) + # 游标移到消息列表末尾（全量压缩后无未压缩消息） + last_msg_idx = len(all_messages) - 1 if all_messages else -1 + update_session_meta(session_id, agent_name, last_msg_idx, new_compressed, last_total_tokens) + meta["cursor_index"] = last_msg_idx + meta["compressed_content"] = new_compressed + + # 构建返回消息列表 + system_text = system_prompt + if meta["compressed_content"]: + system_text += f"\n\n【历史对话摘要】\n{meta['compressed_content']}" + + messages = [SystemMessage(content=system_text)] + # 添加游标之后的未压缩消息 + recent = _get_messages_since_cursor(history, meta["cursor_index"]) + messages.extend(recent) + # 添加当前用户消息 + messages.append(HumanMessage(content=user_content)) + + return messages, False + +# ---------- 辅助函数 ---------- def assistant_text(value) -> str: - """把网关返回的 content、reasoning 等（str 或块列表）收成一段可见文本。""" + """把网关返回的 content、reasoning 等收成一段可见文本。""" if value is None: return "" if isinstance(value, str): @@ -95,87 +279,4 @@ def assistant_text(value) -> str: return json.dumps(value, ensure_ascii=False) except TypeError: return str(value).strip() - return str(value).strip() - - -def normalize_user_text_content(value) -> str: - """尽量把用户输入收敛为纯文本，避免把 dict/list 结构直接送进主模型。""" - if value is None: - return "" - if isinstance(value, str): - return value.strip() - if isinstance(value, dict): - for key in ("text", "content", "input", "query", "question", "message"): - v = value.get(key) - if isinstance(v, str) and v.strip(): - return v.strip() - content = value.get("content") - if content is not None: - text = normalize_user_text_content(content) - if text: - return text - return assistant_text(value) - if isinstance(value, list): - parts: list[str] = [] - for item in value: - if isinstance(item, str) and item.strip(): - parts.append(item.strip()) - elif isinstance(item, dict): - if item.get("type") == "text": - text = (item.get("text") or "").strip() - if text: - parts.append(text) - else: - text = normalize_user_text_content(item) - if text: - parts.append(text) - return "\n".join(parts).strip() - return str(value).strip() - - -def extract_result_from_invoke(value) -> str: - """兼容不同 invoke 返回结构，尽量提取可显示的助手文本。""" - if value is None: - return "" - - if hasattr(value, "content"): - text = assistant_text(getattr(value, "content", None)) - if text: - return text - - if isinstance(value, dict): - for key in ("output", "content", "text", "answer", "result"): - text = assistant_text(value.get(key)) - if text: - return text - - message = value.get("message") - if message is not None: - text = extract_result_from_invoke(message) - if text: - return text - - generations = value.get("generations") - if isinstance(generations, list): - for generation in generations: - text = extract_result_from_invoke(generation) - if text: - return text - - if isinstance(value, list): - for item in value: - text = extract_result_from_invoke(item) - if text: - return text - - if hasattr(value, "message"): - text = extract_result_from_invoke(getattr(value, "message", None)) - if text: - return text - - if hasattr(value, "additional_kwargs"): - text = assistant_text(getattr(value, "additional_kwargs", {}).get("reasoning_content")) - if text: - return text - - return "" \ No newline at end of file + return str(value).strip() \ No newline at end of file diff --git a/src/agent/utils/env_utils.py b/src/agent/utils/env_utils.py index d688f1e..fb675f3 100644 --- a/src/agent/utils/env_utils.py +++ b/src/agent/utils/env_utils.py @@ -28,6 +28,13 @@ LLM_TIMEOUT = _get_env("LLM_TIMEOUT", default="60", cast=float) TTS_BASE_URL = _get_env("TTS_BASE_URL") ASR_BASE_URL = _get_env("ASR_BASE_URL") RAG_BASE_URL = _get_env("RAG_BASE_URL") +LLM_CONTEXT_WINDOW = int(os.getenv("LLM_CONTEXT_WINDOW", 131072)) +MINI_COMPRESS_RATE = float(os.getenv("MINI_COMPRESS_RATE", 0.5)) +HARD_COMPRESS_RATE = float(os.getenv("HARD_COMPRESS_RATE", 0.75)) +CURRENT_TURNS = int(os.getenv("CURRENT_TURNS", 4)) +IMG_SIZE = int(os.getenv("IMG_SIZE", 768)) +RAG_CHUNK_SIZE=int(os.getenv("RAG_CHUNK_SIZE", 500)) +RAG_CHUNK_OVERLAP=int(os.getenv("RAG_CHUNK_OVERLAP", 50)) BROWSER_MAP: Dict[str, Tuple[str, str]] = { "edge": ("microsoft-edge", "Microsoft Edge"), diff --git a/src/agent/utils/image_utils.py b/src/agent/utils/image_utils.py index 58282c8..55e92b4 100644 --- a/src/agent/utils/image_utils.py +++ b/src/agent/utils/image_utils.py @@ -1,13 +1,11 @@ """图片缩放、体积控制与 OpenAI 风格多模态 image_url 片段（供 LLM / Agent 使用）。""" -from __future__ import annotations - import base64 import io import os from typing import Any, Protocol - from PIL import Image +from agent.utils.env_utils import IMG_SIZE class SupportsChatCompletion(Protocol): """具备 ``chat_completion(messages, **kwargs)`` 的 LLM（如 ``ChatQwen35OpenAI``）。""" @@ -26,7 +24,7 @@ def guess_image_mime(extension_without_dot: str) -> str: return "image/jpeg" -def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = 768) -> bytes: +def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = IMG_SIZE) -> bytes: """ 控制图片体积，避免 data-url base64 触发超长 token。 - 若图片尺寸过大，等比例缩放（默认最大边 max_size） @@ -56,9 +54,6 @@ def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = new_width = int(original_width * ratio) new_height = int(original_height * ratio) img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) - print(f"图片已从 {original_width}x{original_height} 缩放到 {new_width}x{new_height}") - else: - print(f"图片尺寸 {original_width}x{original_height} 未超过限制，无需缩放") img_buffer = io.BytesIO() save_format = (original_format or "PNG").upper() @@ -87,7 +82,6 @@ def image_to_base64(image_content: str | bytes) -> str | None: processed = resize_image_if_needed(image_content) return base64.b64encode(processed).decode("utf-8") except Exception as e: - print(f"处理图片 {image_content!r} 时出错: {e}") return None @@ -111,7 +105,6 @@ def image_bytes_to_openai_image_url_part( try: processed = resize_image_if_needed(image_bytes) except Exception as e: - print(f"处理图片字节时出错: {e}") return None mime = _mime_from_magic(processed) b64_str = base64.b64encode(processed).decode("utf-8") diff --git a/src/agent/web/__pycache__/__init__.cpython-312.pyc b/src/agent/web/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index df896fb..0000000 Binary files a/src/agent/web/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/src/agent/web/__pycache__/server.cpython-312.pyc b/src/agent/web/__pycache__/server.cpython-312.pyc deleted file mode 100644 index 2e66f19..0000000 Binary files a/src/agent/web/__pycache__/server.cpython-312.pyc and /dev/null differ diff --git a/src/agent/web/__pycache__/ui_session.cpython-312.pyc b/src/agent/web/__pycache__/ui_session.cpython-312.pyc deleted file mode 100644 index 0e461d7..0000000 Binary files a/src/agent/web/__pycache__/ui_session.cpython-312.pyc and /dev/null differ diff --git a/src/agent/web/app.css b/src/agent/web/app.css index babcdac..c6a04e5 100644 --- a/src/agent/web/app.css +++ b/src/agent/web/app.css @@ -6,7 +6,6 @@ --radius: 14px; --sidebar-w: 272px; --rail-w: 44px; - /* 背景：景深模糊（px），由 JS 写入 html 行内或仍用默认 */ --bg-dof: 0px; --bg-vignette: 0.35; --bubble-user: rgba(236, 254, 255, 0.55); @@ -40,7 +39,7 @@ body { overflow-x: hidden; } -/* 全屏背景 + 景深（对底图做轻微放大以裁掉模糊边缘） */ +/* 全屏背景 + 景深 */ .bg-scene { position: fixed; inset: 0; @@ -52,7 +51,6 @@ body { .bg-image { position: absolute; inset: 0; - /* 默认 cover；加载后 app.js 按 natural 尺寸与视口写死 px，等价 cover + 景深 bleed */ background-color: #334155; background-image: url("/image/background.jpg"); background-size: cover; @@ -435,6 +433,7 @@ body { background: transparent; } +/* ===== 消息容器 ===== */ .msg { max-width: 95%; width: fit-content; @@ -444,19 +443,24 @@ body { gap: 0.35rem; } +/* 用户消息：靠右并限制最大宽度 */ .msg.user { align-self: flex-end; align-items: flex-end; + max-width: 20%; } +/* 助手消息可稍宽 */ .msg.assistant { align-self: flex-start; + max-width: 80%; } +/* ===== 聊天气泡 ===== */ .msg-bubble { - border-radius: 14px; - padding: 0.6rem 0.85rem; - border: 1px solid rgba(148, 163, 184, 0.45); + border-radius: 18px; /* 更明显的圆角 */ + padding: 0.85rem 1.05rem; /* 更充足的内边距，让内容不贴边 */ + border: 1px solid rgba(148, 163, 184, 0.4); line-height: 1.55; white-space: pre-wrap; word-break: break-word; @@ -466,49 +470,65 @@ body { backdrop-filter: blur(12px); -webkit-backdrop-filter: blur(12px); box-shadow: 0 4px 20px rgba(15, 23, 42, 0.06); + overflow: hidden; /* 裁剪图片/代码块等内容的直角，保持圆角边界 */ } .msg.user .msg-bubble { background: var(--bubble-user); + text-align: right; +} + +.msg.user .msg-bubble img { + margin-left: auto; + margin-right: 0; } .msg.assistant .msg-bubble { background: var(--bubble-assistant); } +/* 图片在气泡内留出呼吸空间 */ .msg-bubble img { display: block; width: auto; height: auto; - max-width: min(360px, 100%); - border-radius: 10px; - border: 1px solid rgba(148, 163, 184, 0.35); - margin-top: 0.25rem; + max-width: min(340px, 100%); /* 缩小占比，不撑满气泡 */ + border-radius: 12px; /* 图片自身圆角，配合气泡圆角 */ + border: 1px solid rgba(148, 163, 184, 0.3); + margin: 0.4rem 0; /* 上下对称间距 */ } .msg-bubble > img:first-child { - margin-top: 0; + margin-top: 0.15rem; /* 图片在顶部时也保留一点间距 */ } - .msg-bubble audio { +.msg-bubble > img:last-child { + margin-bottom: 0.15rem; +} + +.msg-bubble audio { max-width: 100%; display: block; margin-top: 0.25rem; } +/* ===== 纯音频消息 ===== */ .msg--audio-only { - max-width: min(var(--audio-bubble-max), 82%); + max-width: min(var(--audio-bubble-max), 72%); } +/* 纯音频消息的气泡：透明无边框，保留音频 pill 尾部空间 */ .msg--audio-only .msg-bubble { - padding: 0.2rem 0; + padding: 0.4rem 0.35rem; /* 给 pill 两侧留出呼吸空间 */ border: none; background: transparent; box-shadow: none; backdrop-filter: none; -webkit-backdrop-filter: none; + overflow: visible; /* 允许音频 pill 尾部小三角超出气泡 */ } +/* ===== 音频胶囊 ===== */ .audio-pill-wrap { display: flex; align-items: center; @@ -764,11 +784,12 @@ body { color: #115e59; } +/* ===== 富文本消息 ===== */ .msg-bubble.msg-bubble--rich { white-space: normal; } - .msg-prose { +.msg-prose { margin: 0.4rem 0; font-size: 0.95rem; } @@ -777,7 +798,7 @@ body { line-height: 1.65; } - .markdown-body p, +.markdown-body p, .markdown-body ul, .markdown-body ol, .markdown-body blockquote, @@ -826,7 +847,8 @@ body { margin-bottom: 0; } - .msg-typing .msg-typing-bubble { +/* ===== 打字动画 ===== */ +.msg-typing .msg-typing-bubble { min-width: 5.5rem; padding: 0.65rem 1rem; } @@ -860,33 +882,7 @@ body { animation-delay: 0.3s; } -@keyframes audioWave { - 0%, - 100% { - transform: scaleY(0.65); - opacity: 0.72; - } - - 50% { - transform: scaleY(1.15); - opacity: 1; - } -} - -@keyframes typingBounce { - 0%, - 60%, - 100% { - opacity: 0.25; - transform: translateY(0); - } - - 30% { - opacity: 1; - transform: translateY(-3px); - } -} - +/* ===== 代码块 ===== */ .code-block-wrap { margin: 0.55rem 0; border-radius: 10px; @@ -949,6 +945,7 @@ body { white-space: pre; } +/* ===== 输入区域 ===== */ .composer { flex-shrink: 0; margin-top: 0.65rem; @@ -1072,6 +1069,17 @@ body { filter: hue-rotate(315deg) saturate(1.2); } +.mic-btn.is-auto-input { + border-color: rgba(148, 163, 184, 0.35); + background: rgba(203, 213, 224, 0.65); + box-shadow: none; +} + +.mic-btn.is-auto-input img { + filter: grayscale(100%); + opacity: 0.65; +} + .mm-input textarea::placeholder { color: #64748b; } @@ -1088,6 +1096,27 @@ body { align-items: center; } +.composer-auto-input { + display: flex; + align-items: center; + margin-left: auto; + gap: 0.5rem; + padding: 0.35rem 0; +} + +.auto-input-label { + font-size: 0.85rem; + color: var(--muted); + cursor: pointer; +} + +.auto-input-check { + width: 1.2rem; + height: 1.2rem; + cursor: pointer; + accent-color: var(--accent); +} + .composer-send { display: flex; flex-direction: column; @@ -1127,7 +1156,7 @@ body { cursor: not-allowed; } - .btn-secondary { +.btn-secondary { background: rgba(255, 255, 255, 0.35); } @@ -1145,7 +1174,7 @@ body { background: rgba(255, 255, 255, 0.2); } - .file-btn { +.file-btn { display: inline-flex; align-items: center; gap: 0.25rem; @@ -1167,7 +1196,33 @@ body { color: var(--text); } -/* —— 窄屏 / 手机：侧栏改为抽屉，主区全宽，避免 272px 挤爆布局 —— */ +/* ===== 动画 ===== */ +@keyframes audioWave { + 0%, + 100% { + transform: scaleY(0.65); + opacity: 0.72; + } + 50% { + transform: scaleY(1.15); + opacity: 1; + } +} + +@keyframes typingBounce { + 0%, + 60%, + 100% { + opacity: 0.25; + transform: translateY(0); + } + 30% { + opacity: 1; + transform: translateY(-3px); + } +} + +/* ===== 窄屏 / 手机 ===== */ @media (max-width: 768px) { .main-header { align-items: center; @@ -1177,6 +1232,11 @@ body { display: inline-flex; } + /* 用户消息在移动端适度放宽，避免 20% 过窄 */ + .msg.user { + max-width: 45%; + } + .app-root.is-mobile .sidebar { position: fixed; z-index: 50; @@ -1266,4 +1326,4 @@ body { .chat-messages { -webkit-overflow-scrolling: touch; } -} +} \ No newline at end of file diff --git a/src/agent/web/app.js b/src/agent/web/app.js index bf9e8d7..354db3a 100644 --- a/src/agent/web/app.js +++ b/src/agent/web/app.js @@ -8,6 +8,7 @@ const imageInput = $("imageInput"); const audioInput = $("audioInput"); const micBtn = $("micBtn"); + const autoInputCheck = $("autoInputCheck"); const attachmentPreview = $("attachmentPreview"); const sendBtn = $("sendBtn"); const clearBtn = $("clearBtn"); @@ -47,7 +48,21 @@ let recorderStream = null; let recordedChunks = []; let recordingMimeType = ""; - /** 最近一次 renderMessages 的 JSON 可序列化快照，用于发送前拼接带 blob: 的乐观用户消息 */ + // 自动语音输入相关变量 + let autoInputEnabled = false; + let audioContext = null; + let analyser = null; + let dataArray = null; + let mediaStreamSource = null; + let autoInputStream = null; + let autoRecordingTimeout = null; + let silenceTimeout = null; + // 声音阈值配置（已优化） + const START_THRESHOLD = 0.18; // 高于此开始录音（较高阈值） + const STOP_THRESHOLD = 0.06; // 低于此停止录音（较低阈值，接近静音） + const AUTO_START_DELAY = 100; + const AUTO_STOP_DELAY = 800; // 缩短停止延迟 + /** 最近一次 renderMessages 的 JSON 可序列化快照 */ let lastRenderedMessages = []; let activeAudioController = null; @@ -113,7 +128,6 @@ .replace(/"/g, """); } - /** 与后端气泡结构一致：纯文字为 string，否则为 parts 数组（顺序：图片 → 文字 → 音频） */ function buildOptimisticUserContent(text, imageBlobUrls, audioPart) { const parts = []; (imageBlobUrls || []).forEach((url) => { @@ -194,12 +208,179 @@ recorderStream = null; } + function stopAudioContext() { + if (audioContext) { + try { + audioContext.close(); + } catch { + /* ignore */ + } + audioContext = null; + analyser = null; + dataArray = null; + mediaStreamSource = null; + } + } + + // 改用 RMS 计算音量，更符合物理响度 + function calculateVolume() { + if (!analyser || !dataArray) return 0; + const bufferLength = analyser.fftSize; + const timeData = new Float32Array(bufferLength); + analyser.getFloatTimeDomainData(timeData); + let sumSquares = 0; + for (let i = 0; i < bufferLength; i++) { + sumSquares += timeData[i] * timeData[i]; + } + return Math.sqrt(sumSquares / bufferLength); + } + + function handleVolumeDetection() { + if (!autoInputEnabled || !analyser) return; + + const volume = calculateVolume(); + + handleVolumeDetection.frameCount = (handleVolumeDetection.frameCount || 0) + 1; + if (handleVolumeDetection.frameCount % 10 === 0) { + console.log("音量检测:", volume.toFixed(3), "| 状态:", + mediaRecorder?.state || "idle", "| 自动输入:", autoInputEnabled); + } + + if (volume > START_THRESHOLD) { + console.log("检测到高分贝，音量:", volume.toFixed(3), "> 阈值:", START_THRESHOLD); + if (silenceTimeout) { + clearTimeout(silenceTimeout); + silenceTimeout = null; + } + if (!mediaRecorder && !autoRecordingTimeout) { + console.log("设置延迟启动录音，延迟:", AUTO_START_DELAY, "ms"); + autoRecordingTimeout = setTimeout(() => { + if (autoInputEnabled && !mediaRecorder) { + startMicRecording().catch((e) => { + console.error("自动开始录音失败:", e); + autoRecordingTimeout = null; + }); + } + autoRecordingTimeout = null; + }, AUTO_START_DELAY); + } + } else if (volume < STOP_THRESHOLD && mediaRecorder && mediaRecorder.state === "recording") { + console.log("检测到静音，音量:", volume.toFixed(3), "< 阈值:", STOP_THRESHOLD); + if (autoRecordingTimeout) { + clearTimeout(autoRecordingTimeout); + autoRecordingTimeout = null; + } + if (!silenceTimeout) { + console.log("设置延迟停止录音，延迟:", AUTO_STOP_DELAY, "ms"); + silenceTimeout = setTimeout(() => { + if (mediaRecorder && mediaRecorder.state === "recording") { + stopMicRecording().catch((e) => { + console.error("自动停止录音失败:", e); + }); + } + silenceTimeout = null; + }, AUTO_STOP_DELAY); + } + } + + if (autoInputEnabled) { + requestAnimationFrame(handleVolumeDetection); + } + } + + async function startAutoInput() { + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + throw new Error("当前浏览器不支持麦克风"); + } + + audioContext = new (window.AudioContext || window.webkitAudioContext)(); + + if (audioContext.state === 'suspended') { + await audioContext.resume(); + } + + analyser = audioContext.createAnalyser(); + analyser.fftSize = 512; // 平衡速度与精度 + dataArray = new Float32Array(analyser.fftSize); // 时域数据 + mediaStreamSource = null; // 稍后连接 + + autoInputStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaStreamSource = audioContext.createMediaStreamSource(autoInputStream); + mediaStreamSource.connect(analyser); + + console.log("自动语音输入已开启，音频上下文状态:", audioContext.state); + console.log("音量检测阈值 - 开始:", START_THRESHOLD, "停止:", STOP_THRESHOLD); + + handleVolumeDetection(); + } + + function stopAutoInput() { + if (autoRecordingTimeout) { + clearTimeout(autoRecordingTimeout); + autoRecordingTimeout = null; + } + if (silenceTimeout) { + clearTimeout(silenceTimeout); + silenceTimeout = null; + } + if (autoInputStream) { + autoInputStream.getTracks().forEach((track) => { + try { + track.stop(); + } catch { + /* ignore */ + } + }); + autoInputStream = null; + } + stopAudioContext(); + } + + function setAutoInputUi(enabled) { + if (!micBtn) return; + micBtn.classList.toggle("is-auto-input", enabled); + micBtn.disabled = enabled; + micBtn.title = enabled ? "自动语音输入中（不可用）" : "使用麦克风录音"; + micBtn.setAttribute("aria-label", enabled ? "自动语音输入中（不可用）" : "使用麦克风录音"); + } + + function toggleAutoInput(enabled) { + autoInputEnabled = enabled; + + if (enabled) { + setAutoInputUi(true); + startAutoInput().catch((e) => { + console.error("开启自动语音输入失败:", e); + autoInputEnabled = false; + setAutoInputUi(false); + if (autoInputCheck) autoInputCheck.checked = false; + alert(e.message || "开启自动语音输入失败"); + }); + } else { + setAutoInputUi(false); + stopAutoInput(); + if (mediaRecorder && mediaRecorder.state === "recording") { + stopMicRecording().catch(() => {}); + } + } + } + function setRecordingUi(recording) { if (!micBtn) return; micBtn.classList.toggle("is-recording", recording); micBtn.setAttribute("aria-pressed", recording ? "true" : "false"); - micBtn.title = recording ? "停止录音" : "使用麦克风录音"; - micBtn.setAttribute("aria-label", recording ? "停止录音" : "使用麦克风录音"); + + if (recording) { + micBtn.title = "停止录音"; + micBtn.setAttribute("aria-label", "停止录音"); + micBtn.classList.remove("is-auto-input"); + } else { + micBtn.title = autoInputEnabled ? "自动语音输入中（不可用）" : "使用麦克风录音"; + micBtn.setAttribute("aria-label", autoInputEnabled ? "自动语音输入中（不可用）" : "使用麦克风录音"); + if (autoInputEnabled) { + micBtn.classList.add("is-auto-input"); + } + } } function inferRecordedExtension(mimeType) { @@ -234,7 +415,11 @@ recordedChunks = []; recordingMimeType = ""; mediaRecorder = null; - stopRecorderStream(); + if (!autoInputEnabled) { + stopRecorderStream(); + } else { + recorderStream = null; + } setRecordingUi(false); } }, { once: true }); @@ -242,7 +427,11 @@ recordedChunks = []; recordingMimeType = ""; mediaRecorder = null; - stopRecorderStream(); + if (!autoInputEnabled) { + stopRecorderStream(); + } else { + recorderStream = null; + } setRecordingUi(false); reject(event.error || new Error("录音失败")); }, { once: true }); @@ -273,7 +462,12 @@ } }) || ""; - recorderStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (autoInputEnabled && autoInputStream) { + recorderStream = autoInputStream; + } else { + recorderStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } + recordedChunks = []; recordingMimeType = supportedType; mediaRecorder = supportedType @@ -433,7 +627,6 @@ return wrap; } - /** 解析 ```lang … ``` 与正文，生成片段（用于助手气泡）。 */ function parseAssistantContent(full) { return renderMarkdownToFragment(full); } @@ -445,7 +638,6 @@ bubble.appendChild(renderMarkdownToFragment(t)); } - /** 整包到达后：先伪流式显示纯文本，再替换为代码块等富结构。 */ async function pseudoStreamAssistant(bubble, full) { const t = full == null ? "" : String(full); bubble.classList.add("msg-bubble--rich", "markdown-body"); @@ -489,7 +681,6 @@ }); } - /** 流式渲染新生成的 assistant 消息（允许自动播放音频） */ async function streamAssistantMessage(msg) { const wrap = document.createElement("div"); wrap.className = "msg assistant" + (isAudioOnlyAssistantMessage(msg) ? " msg--audio-only" : ""); @@ -508,7 +699,6 @@ await pseudoStreamAssistant(bubble, textParts.map((part) => part.text || "").join("\n")); } - // 媒体部分：显式开启 allowAutoplay const mediaOptions = { autoplay: true, compact: textParts.length === 0, @@ -562,7 +752,7 @@ compact = false, messageMeta = null, role = "assistant", - allowAutoplay = false, // 默认不自动播放，只对新生成消息显式开启 + allowAutoplay = false, } = options; const wrap = document.createElement("div"); @@ -680,7 +870,6 @@ wrap.appendChild(button); updateProgressUi(); - // 只有 allowAutoplay 为 true 时才自动播放 const shouldAutoplay = allowAutoplay !== false && Boolean( autoplay || part.autoplay || (messageMeta && messageMeta.autoplay) ); @@ -748,11 +937,6 @@ return div; } - /** - * 渲染单条消息 - * @param {object} msg - 消息对象 - * @param {boolean} allowAutoplay - 是否允许音频自动播放（仅新生成消息为 true） - */ function renderMessage(msg, allowAutoplay = false) { const wrap = document.createElement("div"); wrap.className = "msg " + (msg.role === "user" ? "user" : "assistant") + (isAudioOnlyAssistantMessage(msg) ? " msg--audio-only" : ""); @@ -796,11 +980,6 @@ return wrap; } - /** - * 渲染消息列表 - * @param {Array} messages - 消息数组 - * @param {boolean} allowAutoplay - 是否允许音频自动播放（默认 false） - */ function renderMessages(messages, allowAutoplay = false) { chatMessages.innerHTML = ""; const list = messages || []; @@ -843,7 +1022,7 @@ sessionId = data.sessionId; sessions = data.sessions || []; renderSessions(); - renderMessages(data.messages); // allowAutoplay 默认为 false + renderMessages(data.messages); } async function selectSession(id) { @@ -852,7 +1031,7 @@ sessionId = data.sessionId; sessions = data.sessions || []; renderSessions(); - renderMessages(data.messages); // 切换会话不自动播放 + renderMessages(data.messages); closeDrawerIfMobile(); } @@ -866,7 +1045,7 @@ audioInput.value = ""; renderSessions(); renderAttachmentChips(); - renderMessages(data.messages); // 新建会话不自动播放 + renderMessages(data.messages); closeDrawerIfMobile(); } @@ -885,7 +1064,7 @@ audioInput.value = ""; renderAttachmentChips(); renderSessions(); - renderMessages(data.messages || []); // 删除后不自动播放 + renderMessages(data.messages || []); closeDrawerIfMobile(); } @@ -894,7 +1073,7 @@ const fd = new FormData(); fd.append("session_id", sessionId); const data = await api("/api/sessions/clear", { method: "POST", body: fd }); - renderMessages(data.messages); // 清空后不自动播放 + renderMessages(data.messages); } async function sendChat() { @@ -925,7 +1104,7 @@ } const optimisticContent = buildOptimisticUserContent(trimmed, imageBlobUrls, audioPart); if (optimisticContent != null) { - renderMessages([...priorMessages, { role: "user", content: optimisticContent }]); // 乐观渲染不自动播放 + renderMessages([...priorMessages, { role: "user", content: optimisticContent }]); } textInput.value = ""; pendingImages = []; @@ -955,7 +1134,7 @@ renderAttachmentChips(); } renderSessions(); - renderMessages(startData.messages || []); // start 返回不自动播放 + renderMessages(startData.messages || []); revokeBlobUrlStack(blobUrls); typingEl = appendThinkingIndicator(); @@ -974,11 +1153,11 @@ const last = messages[messages.length - 1]; if (last && last.role === "assistant") { const head = messages.slice(0, -1); - renderMessages(head); // 历史消息不自动播放 + renderMessages(head); revokeBlobUrlStack(blobUrls); - await streamAssistantMessage(last); // 只有新消息流式渲染时自动播放 + await streamAssistantMessage(last); } else { - renderMessages(messages); // 无 assistant 消息时不自动播放 + renderMessages(messages); revokeBlobUrlStack(blobUrls); } } catch (e) { @@ -988,7 +1167,7 @@ pendingImages = savedImages.slice(); pendingAudio = savedAudio; renderAttachmentChips(); - renderMessages(priorMessages); // 回滚不自动播放 + renderMessages(priorMessages); } if (typingEl && typingEl.parentNode) typingEl.remove(); alert(e.message || String(e)); @@ -998,12 +1177,12 @@ } } - imageInput.addEventListener("change", () => { const files = Array.from(imageInput.files || []); pendingImages = pendingImages.concat(files); imageInput.value = ""; renderAttachmentChips(); + textInput.focus(); }); audioInput.addEventListener("change", () => { @@ -1011,6 +1190,7 @@ pendingAudio = f || null; audioInput.value = ""; renderAttachmentChips(); + textInput.focus(); }); window.addEventListener("beforeunload", () => { @@ -1033,6 +1213,7 @@ sendBtn.addEventListener("click", sendChat); micBtn?.addEventListener("click", () => toggleMicRecording().catch((e) => alert(e.message || String(e)))); + autoInputCheck?.addEventListener("change", (ev) => toggleAutoInput(ev.target.checked)); clearBtn.addEventListener("click", () => clearChat().catch((e) => alert(e.message))); newSessionBtn.addEventListener("click", () => newSession().catch((e) => alert(e.message))); deleteSessionBtn?.addEventListener("click", () => deleteSession().catch((e) => alert(e.message))); @@ -1087,14 +1268,13 @@ } }); } - loadBgPrefs(); - /** 背景图原始像素（预加载成功后写入） */ let bgNaturalW = 0; let bgNaturalH = 0; - /** @type {ReturnType | null} */ let bgFitTimer = null; + loadBgPrefs(); + function backgroundImageHref() { try { return new URL("image/background.jpg", window.location.href).href; @@ -1109,7 +1289,6 @@ return Number.isFinite(n) ? n : 0; } - /** 按「铺满视口」缩放（等价 cover，无黑边；景深略放大以减轻模糊露边） */ function applyBackgroundFit() { if (!bgImageLayer || !bgNaturalW || !bgNaturalH) return; const vw = window.innerWidth; @@ -1131,7 +1310,6 @@ }, 100); } - /** 与 CSS 中 url("background.jpg") 一致；预加载得到尺寸后再写 background-size */ function wireBackgroundImage() { if (!bgImageLayer) return; const href = backgroundImageHref(); diff --git a/src/agent/web/index.html b/src/agent/web/index.html index 38efb5b..9a8d649 100644 --- a/src/agent/web/index.html +++ b/src/agent/web/index.html @@ -115,6 +115,13 @@ + +
+ + +
diff --git a/src/agent/web/ui_session.py b/src/agent/web/ui_session.py index 15f98e0..d99e41b 100644 --- a/src/agent/web/ui_session.py +++ b/src/agent/web/ui_session.py @@ -15,7 +15,6 @@ from agent.utils.agent_utils import ( CHAT_HISTORY_DB_FS, MULTI_INPUTS_ROOT, assistant_text, - extract_result_from_invoke, ) from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS from agent.chatbot_agent import get_history @@ -98,72 +97,34 @@ def _load_ui_messages_from_db(session_id: str) -> list: if m.type == "human": rows.append({"role": "user", "content": _lc_content_for_ui_messages(m.content)}) elif m.type in ("ai", "assistant"): - text = assistant_text(m.content) or extract_result_from_invoke(m) + text = assistant_text(m.content) rows.append({"role": "assistant", "content": text or "（空回复）"}) return rows def _build_user_chat_content(query) -> str | list | None: + """根据上传文件扩展名直接生成多模态 content（与 LLM 输入格式一致）""" parts: list = [] for filepath in query.get("files", []) or []: - if filepath: + if not filepath: + continue + ext = os.path.splitext(filepath)[1].lower() + if ext in IMAGE_FILE_EXTENSIONS: + parts.append({"type": "image_url", "image_url": {"url": filepath}}) + elif ext in _AUDIO_EXTENSIONS: + parts.append({"type": "audio", "audio_path": filepath}) + else: parts.append({"path": filepath}) text = (query.get("text") or "").strip() if text: parts.append({"type": "text", "text": text}) if not parts: return None - if len(parts) == 1 and isinstance(parts[0], dict) and parts[0].get("type") == "text": + if len(parts) == 1 and parts[0].get("type") == "text": return parts[0]["text"] return parts -def collect_media_paths_and_query_text(user_msg: dict) -> tuple[list[str], list[str], str]: - raw = user_msg.get("content") - text_segments: list[str] = [] - image_paths: list[str] = [] - audio_paths: list[str] = [] - - if isinstance(raw, str): - t = raw.strip() - if t: - text_segments.append(t) - return image_paths, audio_paths, "\n".join(text_segments).strip() - - if not isinstance(raw, list): - return image_paths, audio_paths, "" - - for piece in raw: - if isinstance(piece, str): - t = piece.strip() - if t: - text_segments.append(t) - continue - if not isinstance(piece, dict): - continue - if piece.get("type") == "text": - text_content = (piece.get("text") or "").strip() - if text_content: - text_segments.append(text_content) - continue - file_path = piece.get("path") or (piece.get("file") or {}).get("path") - if piece.get("type") == "image_url": - file_path = file_path or (piece.get("image_url") or {}).get("url") - if piece.get("type") == "audio": - file_path = file_path or piece.get("audio_path") - if not file_path: - continue - ext = os.path.splitext(file_path)[1].lower() - if ext in IMAGE_FILE_EXTENSIONS: - image_paths.append(file_path) - elif ext in _AUDIO_EXTENSIONS: - audio_paths.append(file_path) - else: - text_segments.append(f"[用户上传了附件文件: {os.path.basename(file_path)}]") - - return image_paths, audio_paths, "\n".join(text_segments).strip() - - def rewrite_user_message_files_to_session_dir(user_msg: dict, session_id: str) -> None: dest_dir = session_dir(session_id) os.makedirs(dest_dir, exist_ok=True) @@ -304,7 +265,7 @@ def _serialize_assistant_entry(assistant_msg: dict | None) -> dict: if isinstance(ac, str): atext = ac else: - atext = assistant_text(ac) or extract_result_from_invoke(assistant_msg) or "（空回复）" + atext = assistant_text(ac) or "（空回复）" audio_relpath = None audio = meta.get("audio") if isinstance(audio, dict) and audio.get("path"): @@ -458,4 +419,4 @@ def clear_ui_and_manifest(session_id: str): os.makedirs(os.path.dirname(mp), exist_ok=True) with open(mp, "w", encoding="utf-8") as f: json.dump({"version": 1, "turns": []}, f, ensure_ascii=False, indent=2) - return [], {"text": "", "files": []} + return [], {"text": "", "files": []} \ No newline at end of file