目前agent infra及rag框架已经搭好

.env

@@ -16,8 +16,7 @@ RAG_BASE_URL = "http://192.168.137.96:8903"
 #6000pro：
 LLM_API_KEY='semi-ai.tech'
 LLM_BASE_URL="http://semi-ai.tech:40056/v1"
-LLM_MODEL = "Intel/Qwen3.6-27B-int4-AutoRound"
-
+LLM_MODEL = "Intel/Qwen3.6-35B-A3B-int4-mixed-AutoRound"
 
 #A100：
 # LLM_API_KEY = 'sk-kGhWAQTqLk9ECsmIDb758a456bD34e92B9771dF2E36432Fb'
@@ -27,3 +26,12 @@ LLM_MODEL = "Intel/Qwen3.6-27B-int4-AutoRound"
 # LLM_REQUESTS_PER_SECOND = 10.0
 # LLM_MAX_BUCKET_SIZE = 10
 # LLM_CHECK_EVERY_N_SECONDS = 0.1
+
+#context:
+LLM_CONTEXT_WINDOW=32768
+MINI_COMPRESS_RATE=0.4
+HARD_COMPRESS_RATE=0.6
+CURRENT_TURNS=4
+IMG_SIZE=768
+RAG_CHUNK_SIZE=500
+RAG_CHUNK_OVERLAP=50

README.md

@@ -1,32 +1,36 @@
-# 进入项目目录并提交代码（Windows 下路径为 .\Agent\）
+# Agent Web Chat
+
+基于 LangGraph 的多模态对话 Agent，支持文本、图片、音频输入，具备屏幕截图、浏览器控制、RAG 知识库、数据库查询等工具调用能力。
+
+## 快速开始
+
+### 1. 克隆仓库并进入项目目录
+
+```bash
+git clone <your-repo-url>
 cd Agent
-git add .
-git commit -m "修改内容的说明"
-git push
 
-# 创建并激活虚拟环境（Linux/macOS 用 source，Windows 用 myenv\Scripts\activate）
+Windows:
+bash
 python -m venv myenv
-source myenv/bin/activate || myenv/Scripts/activate
+.\myenv\Scripts\activate
+macOS / Linux:
+bash
+python -m venv myenv
+source myenv/bin/activate
 
-# 升级 pip 并安装核心依赖
-python -m pip install --upgrade pip
-pip install -U langchain langchain-openai python-dotenv
-
-# 以可编辑模式安装当前项目（配置见 pyproject.toml）
+pip install --upgrade pip
+pip install -r requirements.txt
 pip install -e .
 
-# 安装 LangGraph CLI（开发模式）
-pip install "langgraph-cli[inmem]"
-
-# 可选：安装数据库相关依赖（psycopg2-binary 用于 PostgreSQL）
-pip install sqlalchemy pymysql loguru psycopg2-binary
-
-# 环境变量配置
-# Copy .env.example to .env and fill in the required values before running the app.
-cp .env.example .env
-
-# 必须填写的变量：
-# .env.example
+gedit .env
+必须填写的变量：
+LLM_BASE_URL – 大模型 API 基础地址
+LLM_API_KEY – API 密钥
+LLM_MODEL – 模型名称（如 Qwen/Qwen2.5-72B-Instruct）
+TTS_BASE_URL – 文本转语音服务地址（可选，若不使用音频回复可不填）
+ASR_BASE_URL – 语音识别服务地址（可选）
+RAG_BASE_URL – RAG 嵌入与重排序服务地址（可选）
 
 # 启动 Web 对话服务（推荐）
 # 确保在 Agent 目录下，且 PYTHONPATH 包含 src 目录
@@ -35,8 +39,11 @@ export PYTHONPATH=src:$PYTHONPATH   # Linux/macOS
 # 或 Windows PowerShell: $env:PYTHONPATH="src;$env:PYTHONPATH"
 uvicorn agent.web.server:app --host 127.0.0.1 --port 8765
 
-# API 为两阶段调用：POST /api/chat/start（落盘用户消息） + POST /api/chat/complete（推理）
-# 前端已按此顺序调用。
-
 # （可选）通过 Cloudflare Tunnel 暴露公网
-cloudflared tunnel --url http://localhost:8765
+cloudflared tunnel --url http://localhost:8765
+
+# 进入项目目录并提交代码（Windows 下路径为 .\Agent\）
+cd Agent
+git add .
+git commit -m "修改内容的说明"
+git push

requirements.txt

@@ -0,0 +1,43 @@
+# Core LangChain & LangGraph
+langchain>=0.1.0
+langchain-core>=0.1.0
+langchain-community>=0.1.0
+langchain-openai>=0.2.0
+langgraph>=1.0.0
+openai>=1.0.0
+
+# Web framework & server
+fastapi>=0.115.0
+uvicorn[standard]>=0.32.0
+python-multipart>=0.0.9
+
+# Database & vector store
+sqlalchemy>=2.0.0
+psycopg2-binary>=2.9.0
+chromadb>=0.4.0
+
+# Audio / TTS / ASR
+pyaudio>=0.2.11
+wave
+pydub>=0.25.1          # for audio format conversion
+requests>=2.31.0
+
+# Screen capture & browser control
+pyautogui>=0.9.54
+webbrowser             # built-in, no need to list
+
+# Image processing
+Pillow>=10.0.0
+
+# Document parsing (RAG)
+pandas>=2.0.0
+python-docx>=1.0.0
+pdfplumber>=0.10.0
+
+# Utilities
+python-dotenv>=1.0.1
+loguru>=0.7.0
+pydantic>=2.0.0
+
+# Optional but often needed
+typing-extensions>=4.0.0

src/agent/chatbot_agent.py

@@ -1,101 +1,45 @@
-from typing import Any, cast, Dict
+# agent/chatbot_agent.py
+from typing import Any, Dict
 
-from langchain_classic.agents import AgentExecutor, create_tool_calling_agent
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.runnables import RunnablePassthrough, RunnableWithMessageHistory
-from langchain_core.messages import HumanMessage
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
+from langchain.agents import create_agent
 
 from agent.llm import Qwen36_35B_A3B_FP8_LLM
 from agent.tools import get_tools
-from agent.utils.agent_utils import assistant_text, get_history, normalize_user_text_content, summarize_history
+from agent.utils.agent_utils import (
+    assistant_text,
+    get_history,
+    summarizer,                       # 使用新的上下文感知构建器
+    get_session_meta,
+    update_session_meta,
+)
 from agent.web.ui_session import (
-    collect_media_paths_and_query_text,
     rewrite_user_message_files_to_session_dir,
     session_dir,
     sync_manifest_from_chat_history,
 )
 from agent.memory.system_prompt import MAIN_SYSTEM_PROMPT
 
-# ---------- 1. 提示模板----------
-prompt = ChatPromptTemplate.from_messages([
-    ("system", "{system_message}"),
-    ("placeholder", "{chat_history}"),
-    MessagesPlaceholder(variable_name="human_message"),   
-    ("placeholder", "{agent_scratchpad}"),
-])
-
+# ---------- 1. 工具 ----------
 tools = get_tools()
 
-# ---------- 2. 创建工具调用 agent ----------
-tool_calling_agent = create_tool_calling_agent(
-    llm=Qwen36_35B_A3B_FP8_LLM.bind(tool_choice="auto"),
+# ---------- 2. 创建 ReAct agent（LangGraph）----------
+agent_graph = create_agent(
+    model=Qwen36_35B_A3B_FP8_LLM,
     tools=tools,
-    prompt=prompt,
+    name="main"
 )
 
-# ---------- 3. agent executor ----------
-agent_executor = AgentExecutor(
-    agent=tool_calling_agent,
-    tools=tools,
-    verbose=True,
-    handle_parsing_errors=True,
-    stream_runnable=False,
-)
+# ---------- 3. 辅助函数：保存消息到数据库 ----------
+def save_messages_to_history(session_id: str, user_msg: HumanMessage, assistant_msg: AIMessage) -> None:
+    """将用户消息和助手消息追加到 SQLite 历史记录中"""
+    if not session_id:
+        return
+    history = get_history(session_id)
+    history.add_message(user_msg)
+    history.add_message(assistant_msg)
 
-# ---------- 4. 带历史记录的链 ----------
-agent_with_history = RunnableWithMessageHistory(
-    runnable=agent_executor,
-    get_session_history=get_history,
-    input_messages_key="human_message",    
-    history_messages_key="chat_history",
-)
-
-# ---------- 5. 注入摘要和系统提示 ----------
-agent_with_history = (
-    RunnablePassthrough.assign(
-        # ✅ 从输入数据中的 agent_config 获取配置
-        summarize_result=lambda x: summarize_history(x["input"], x["agent_config"])
-    )
-    | RunnablePassthrough.assign(
-        input=lambda x: x["input"],
-        chat_history=lambda x: x["summarize_result"]["chat_history"],
-        system_message=lambda x: (
-            MAIN_SYSTEM_PROMPT
-            + (
-                "\n\n【此前对话摘要】\n" + x["summarize_result"]["summary"]
-                if x["summarize_result"]["summary"]
-                else ""
-            )
-        ),
-        human_message=lambda x: [HumanMessage(content=x["input"])],
-    )
-    | agent_with_history
-)
-
-
-# ---------- 6. 辅助函数：将前端 chat_history 转为 LangChain 消息列表 ----------
-def _extract_user_input_parts(user_msg: Dict[str, Any]) -> tuple[list[str], list[str], list[dict[str, Any]] | str]:
-    image_paths, audio_paths, query_text = collect_media_paths_and_query_text(user_msg)
-
-    content_parts: list[dict[str, Any]] = []
-    for img_path in image_paths:
-        content_parts.append({"type": "image_url", "image_url": {"url": img_path}})
-
-    for audio_path in audio_paths:
-        content_parts.append({
-            "type": "audio",
-            "audio_path": audio_path,
-        })
-    if query_text:
-        content_parts.append({"type": "text", "text": query_text})
-
-    if content_parts:
-        if len(content_parts) == 1 and content_parts[0].get("type") == "text":
-            return image_paths, audio_paths, content_parts[0]["text"]
-        return image_paths, audio_paths, content_parts
-    return image_paths, audio_paths, normalize_user_text_content(user_msg.get("content", query_text))
-
-# ---------- 7. 核心执行函数 ----------
+# ---------- 4. 核心执行函数 ----------
 def execute_agent(
     chat_history: list[dict[str, Any]],
     session_id: str = "test",
@@ -106,34 +50,65 @@ def execute_agent(
     if not chat_history or chat_history[-1].get("role") != "user":
         return chat_history
 
+    # 上传文件处理
     if session_id:
         rewrite_user_message_files_to_session_dir(chat_history[-1], session_id)
 
-    _image_paths, _audio_paths, user_content = _extract_user_input_parts(chat_history[-1])
+    # 直接使用 chat_history 中已经正确格式化的 content（由 _build_user_chat_content 生成）
+    user_content = chat_history[-1]["content"]
 
-    # 构建运行时配置
-    config = {
-        "configurable": {
-            "session_id": session_id,
-        },
+    # 动态上下文感知构建消息
+    messages, context_exceeded = summarizer(
+        session_id=session_id,
+        agent_name="main",                  # 固定 agent 名称，可扩展
+        system_prompt=MAIN_SYSTEM_PROMPT,
+        user_content=user_content,
+        llm=Qwen36_35B_A3B_FP8_LLM,
+    )
+
+    if context_exceeded:
+        # 上下文已满，返回提示信息
+        chat_history.append({
+            "role": "assistant",
+            "content": "上下文已满，请新建会话继续对话。",
+        })
+        if session_id:
+            sync_manifest_from_chat_history(session_id, chat_history, "")
+        return chat_history
+
+    # 调用 graph 的配置
+    graph_config = {
+        "configurable": {"session_id": session_id},
         "metadata": {
             "output_type": output_type,
             "audio_output_dir": session_dir(session_id) if session_id else None,
         },
-        "llm": Qwen36_35B_A3B_FP8_LLM, 
-    }
-
-    invoke_input = {
-        "input": user_content,
-        "agent_config": config,
     }
 
     try:
-        result = cast(dict[str, Any], agent_with_history.invoke(invoke_input, config))
+        # 调用 LangGraph agent
+        result = agent_graph.invoke({"messages": messages}, config=graph_config)
 
-        text_content = assistant_text(result.get("output", ""))
+        # 从最终消息中提取文本回复
+        all_messages = result["messages"]
+        assistant_msg = all_messages[-1] if all_messages else AIMessage(content="")
+        text_content = assistant_text(assistant_msg.content)
 
-        output_type_from_config = config.get("metadata", {}).get("output_type", "text")
+        # 提取 token 使用量并更新到 meta
+        usage = assistant_msg.usage_metadata if hasattr(assistant_msg, 'usage_metadata') else {}
+        total_tokens = usage.get('total_tokens', 0) if isinstance(usage, dict) else 0
+        if total_tokens > 0:
+            meta = get_session_meta(session_id, "main")
+            update_session_meta(
+                session_id=session_id,
+                agent_name="main",
+                cursor_index=meta["cursor_index"],
+                compressed_content=meta["compressed_content"],
+                last_total_tokens=total_tokens,
+            )
+
+        # 处理 TTS 音频数据
+        output_type_from_config = graph_config.get("metadata", {}).get("output_type", "text")
         audio_data = Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs or {}
         if audio_data.get("output_type") != output_type_from_config:
             audio_data = {}
@@ -144,6 +119,7 @@ def execute_agent(
             "audio": audio_data.get("audio"),
         }
 
+        # 根据输出类型构建前端 content
         if output_type_from_config == "audio" and assistant_meta.get("audio"):
             assistant_content = []
         elif output_type_from_config == "both" and assistant_meta.get("audio"):
@@ -151,28 +127,32 @@ def execute_agent(
         else:
             assistant_content = text_content if text_content else "（空回复）"
 
+        # 追加到返回的 chat_history
         chat_history.append({
             "role": "assistant",
             "content": assistant_content,
             "meta": assistant_meta,
         })
 
-        # 用完后清空，避免影响后续无音频的请求
+        # 持久化历史消息
+        save_messages_to_history(
+            session_id,
+            HumanMessage(content=user_content),
+            AIMessage(content=text_content),
+        )
+
+        # 清空音频缓存
         Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs = None
 
     except Exception as e:
-        # 异常处理保持不变
         import traceback
         print("[ERROR] 调用模型时出错:")
         traceback.print_exc()
         err_text = "抱歉，我遇到了一些问题，请稍后再试。"
         chat_history.append({"role": "assistant", "content": err_text})
 
+    # 同步文件清单
     if session_id:
         sync_manifest_from_chat_history(session_id, chat_history, "")
 
-    return chat_history
-
-
-if __name__ == "__main__":
-    pass
+    return chat_history

src/agent/history/multi_inputs/20260514_222254/ui_messages.json

@@ -1,31 +0,0 @@
-{
-  "version": 1,
-  "turns": [
-    {
-      "user": {
-        "parts": [
-          {
-            "kind": "image_url",
-            "relpath": "20260514_222254/b50c460b07654d568e897d92c776675f.jpg"
-          },
-          {
-            "kind": "audio",
-            "relpath": "20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a"
-          },
-          {
-            "kind": "text",
-            "text": "请直接告诉我数量。"
-          }
-        ],
-        "image_description": ""
-      },
-      "assistant": {
-        "text": "图片中有两只猫。",
-        "output_type": "both",
-        "autoplay": false,
-        "audio_duration_seconds": 2.08,
-        "audio_relpath": "20260514_222254/assistant_tts_222333_0.wav"
-      }
-    }
-  ]
-}

src/agent/history/multi_inputs/20260515_130220/ui_messages.json

@@ -1,39 +0,0 @@
-{
-  "version": 2,
-  "turns": [
-    {
-      "user": {
-        "content": [
-          {
-            "type": "text",
-            "text": "你好"
-          }
-        ]
-      },
-      "assistant": {
-        "text": "你好！很高兴为您服务。请问有什么我可以帮您的吗？",
-        "output_type": "text",
-        "autoplay": false,
-        "audio_duration_seconds": null,
-        "audio_relpath": null
-      }
-    },
-    {
-      "user": {
-        "content": [
-          {
-            "type": "text",
-            "text": "现在几点了"
-          }
-        ]
-      },
-      "assistant": {
-        "text": "现在是2026年5月15日 13:02。",
-        "output_type": "text",
-        "autoplay": false,
-        "audio_duration_seconds": null,
-        "audio_relpath": null
-      }
-    }
-  ]
-}

src/agent/llm.py

@@ -1,3 +1,4 @@
+# agent/llm.py
 import io
 import os
 import tempfile
@@ -16,7 +17,7 @@ from langchain_openai import ChatOpenAI
 
 from agent.utils.image_utils import image_bytes_to_openai_image_url_part
 from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, TTS_BASE_URL, ASR_BASE_URL
-from agent.utils.agent_utils import assistant_text, normalize_user_text_content
+from agent.utils.agent_utils import assistant_text
 from agent.utils.image_utils import (
     image_path_to_openai_image_url_part,
     image_bytes_to_openai_image_url_part,
@@ -104,24 +105,31 @@ class ChatQwen(ChatOpenAI):
     def _audio_to_text(self, audio_path: str) -> str:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"音频文件不存在: {audio_path}")
+        
         wav_path = self._convert_audio_to_wav(audio_path)
-        cleanup_wav = wav_path != audio_path
+        cleanup_wav = (wav_path != audio_path)
+        
         try:
-            # 构造 file:// URL（vLLM 要求）
-            abs_path = os.path.abspath(wav_path).replace("\\", "/")
-            audio_url = f"file://{abs_path}"
-
+            # 读取音频文件二进制内容
+            with open(wav_path, "rb") as f:
+                audio_bytes = f.read()
+            
+            # 构建 Data URL
+            import base64
+            b64 = base64.b64encode(audio_bytes).decode("ascii")
+            mime_type = "audio/wav"  # 我们已经转换为 wav
+            data_url = f"data:{mime_type};base64,{b64}"
+            
             payload = {
                 "messages": [
                     {
                         "role": "user",
                         "content": [
-                            {"type": "audio_url", "audio_url": {"url": audio_url}}
+                            {"type": "audio_url", "audio_url": {"url": data_url}}
                         ]
                     }
                 ]
             }
-            # ASR_BASE_URL 末尾可能已带 /v1，统一拼接 /chat/completions
             endpoint = f"{ASR_BASE_URL.rstrip('/')}/chat/completions"
             resp = requests.post(
                 endpoint,
@@ -131,7 +139,6 @@ class ChatQwen(ChatOpenAI):
             )
             resp.raise_for_status()
             result = resp.json()
-            # 提取 assistant 回复文本（音频描述/转录）
             content = result["choices"][0]["message"]["content"]
             return content.strip()
         finally:
@@ -151,7 +158,12 @@ class ChatQwen(ChatOpenAI):
                     except (json.JSONDecodeError, ValueError):
                         pass
             if isinstance(msg, HumanMessage) and isinstance(msg.content, dict):
-                msg = HumanMessage(content=normalize_user_text_content(msg.content), additional_kwargs=msg.additional_kwargs)
+                # 尝试从 dict 中提取文本，否则序列化为字符串
+                text = msg.content.get("text") or msg.content.get("content") or ""
+                if text:
+                    msg = HumanMessage(content=str(text), additional_kwargs=msg.additional_kwargs)
+                else:
+                    msg = HumanMessage(content=json.dumps(msg.content, ensure_ascii=False), additional_kwargs=msg.additional_kwargs)
             elif isinstance(msg, HumanMessage) and isinstance(msg.content, list):
                 new_content: List[Dict[str, Any] | str] = []
                 for part in msg.content:
@@ -179,7 +191,7 @@ class ChatQwen(ChatOpenAI):
                                 else:
                                     new_content.append({"type": "text", "text": f"[用户上传了附件文件: {os.path.basename(file_path)}]"})
                         else:
-                            text = normalize_user_text_content(part)
+                            text = assistant_text(part)
                             if text:
                                 new_content.append({"type": "text", "text": text})
                     elif isinstance(part, str):
@@ -266,7 +278,7 @@ class ChatQwen(ChatOpenAI):
         output_type = metadata.get("output_type", self.output_type)
         audio_output_dir = metadata.get("audio_output_dir")
 
-        # ✅ 移除自定义参数，防止它们传入 OpenAI API
+        # 移除自定义参数，防止它们传入 OpenAI API
         kwargs.pop("output_type", None)
         kwargs.pop("audio_output_dir", None)
 
@@ -354,3 +366,37 @@ class ChatQwen(ChatOpenAI):
 
 
 Qwen36_35B_A3B_FP8_LLM = ChatQwen()
+
+if __name__ == "__main__":
+    import sys
+    
+    # 使用原始字符串避免转义问题
+    TEST_IMAGE_PATH = r"C:\Users\86166\Desktop\image_raw\4.jpg"
+    
+    if not os.path.exists(TEST_IMAGE_PATH):
+        print(f"❌ 图片不存在: {TEST_IMAGE_PATH}")
+        sys.exit(1)
+    
+    # 测试1: 纯图片识别
+    print("=" * 50)
+    print("测试1: 纯图片识别（无文本）")
+    print("=" * 50)
+    result1 = Qwen36_35B_A3B_FP8_LLM.invoke([
+        HumanMessage(content=[{"type": "image_url", "image_url": {"url": TEST_IMAGE_PATH}}])
+    ])
+    print(f"返回类型: {type(result1)}")
+    print(f"返回内容: {result1.content}")
+    print(f"所有属性: {[attr for attr in dir(result1) if not attr.startswith('_')]}")
+    
+    # 测试2: 图片 + 文本
+    print("\n" + "=" * 50)
+    print("测试2: 图文混合输入")
+    print("=" * 50)
+    result2 = Qwen36_35B_A3B_FP8_LLM.invoke([
+        HumanMessage(content=[
+            {"type": "image_url", "image_url": {"url": TEST_IMAGE_PATH}},
+            {"type": "text", "text": "这张图片里有什么？请详细描述"}
+        ])
+    ])
+    print(f"回答: {result2.content}")
+

src/agent/tools/__init__.py

@@ -1,4 +1,5 @@
-from .tool_main import *
+from .tool import *
+from .tool_rag import RAGTool
 
 # from .tool_database_class import *
 # from utils.database_utils import PostgreSQL_Manager
@@ -20,12 +21,12 @@ from .tool_main import *
 # def get_sql_tools():
 #     return _all_database_tools
 
-
 _all_tools = [
     Websearchclass(),
     DateQueryClass(),
     ScreenCaptureDescribeClass(),
-    BrowserControlClass()
+    BrowserControlClass(),
+    RAGTool(), 
 ]
 
 def get_tools():

src/agent/tools/tool.py

@@ -1,54 +1,252 @@
-from langchain.tools import tool
+from __future__ import annotations
 
-@tool("web_search")
-def web_search(query: str) -> str:
-    """
-    一个用于执行网络搜索的工具。它接收一个查询语句作为输入，并返回 Google 的搜索结果。
-    
-    Args:
-        query (str): 搜索查询内容
-    
-    Returns:
-        str: 格式化后的搜索结果
-    """
-    try:
+import io
+from typing import Optional, Literal
+import sys
+import time
+import webbrowser
+import pyautogui
+
+from langchain.tools import BaseTool
+from pydantic import BaseModel, Field, create_model
+
+from agent.llm import Qwen36_35B_A3B_FP8_LLM
+from utils.env_utils import BROWSER_MAP
+
+
+class Websearchclass(BaseTool):
+        
+    args_schema : type[BaseModel] = create_model("Input_type", query=(str, Field(description="搜索查询内容")))
+    name : str = "web_search_class"
+    description : str = "一个网络搜索工具，可以根据用户的查询返回相关的搜索结果。仅当需要查询近期知识或已有知识无法解答需要联网查询时调用，其余任何情况严禁调用。"
+
+    def _get_search_wrapper(self):
         from langchain_community.utilities import GoogleSerperAPIWrapper
-        # 这里直接使用了一个示例的 Serper API 密钥，实际使用时请替换为你自己的密钥
         SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5"
-        # LangChain 官方封装的 Serper API 工具
-        search = GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048)
-        # 直接调用 run 方法获取结果，LangChain 已处理好格式
-        return search.run(query)
-    except Exception as e:
-        return f"执行网络搜索时出错: {str(e)}"   
-    
-@tool("date_query")
-def date_query() -> str:
-    """
-    一个用于执行日期查询的工具。它接收一个查询语句作为输入，并返回相应的日期结果。
-    
-    Returns:
-        str: 格式化后的日期查询结果
-    """
-    try:
-        # 显示当前日期和时间
-        from datetime import datetime
-        now = datetime.now()
-        return now.strftime("%Y-%m-%d %H:%M:%S")
+        return GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048)
 
-    except Exception as e:
-        return f"执行日期查询时出错: {str(e)}"   
+    def _run(self, query: str) -> str:
+        try:
+            return self._get_search_wrapper().run(query)
+        except Exception as e:
+            return f"执行网络搜索时出错: {str(e)}"
+        
+    async def _arun(self, query: str) -> str:
+        try:
+            return await self._get_search_wrapper().arun(query)
+        except Exception as e:
+            return f"执行网络搜索时出错: {str(e)}"
+
+class DateQueryClass(BaseTool):
+
+    # 定义工具的输入参数结构，这里没有输入参数，所以是一个空的模型
+    args_schema : type[BaseModel] = create_model("NoneInput")
+    name : str = "date_query_class"
+    description : str = "一个日期查询工具，可以返回当前的日期信息。仅当需要查询当前时间时调用，其余任何情况严禁调用。"    
+
+    def _run(self) -> str:
+        try:
+            from datetime import datetime
+            now = datetime.now()
+            return now.strftime("%Y-%m-%d %H:%M:%S")
+        except Exception as e:
+            return f"执行日期查询时出错: {str(e)}"
+        
+    async def _arun(self) -> str:
+        return self._run()
+
+
+class ScreenCaptureDescribeClass(BaseTool):
+    """截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。"""
+
+    args_schema: type[BaseModel] = create_model(
+        "ScreenCaptureInput",
+        focus_query=(
+            Optional[str],
+            Field(
+                default=None,
+                description="可选。希望重点观察的界面或问题，例如「错误提示全文」「浏览器地址栏」；留空则整体描述屏幕。",
+            ),
+        ),
+    )
+    name: str = "screen_capture_describe"
+    description: str = (
+        "截取当前主屏幕画面，并用视觉模型转为文字描述。"
+        "注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。"
+    )
+
+    def _run(self, focus_query: Optional[str] = None) -> str:
+        # 1. 确保 pyautogui 可用
+        try:
+            import pyautogui
+        except ImportError as e:
+            return (
+                f"未安装 pyautogui，无法截屏: {e}。"
+                "请在项目环境中执行: pip install pyautogui"
+            )
+
+        # 2. 截屏并转为 PNG 字节流
+        try:
+            pil_img = pyautogui.screenshot()
+            buf = io.BytesIO()
+            pil_img.save(buf, format="PNG")
+            raw_png = buf.getvalue()
+        except Exception as e:
+            return f"截屏失败: {e}"
+
+        # 3. 构建描述指令
+        query = (focus_query or "").strip()
+        
+        # 4. 调用与对话图片上传完全相同的视觉模型进行描述
+        try:
+            # Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义
+            text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query)
+        except Exception as e:
+            return f"屏幕图像视觉解析失败: {e}"
+
+        if not (text or "").strip():
+            return "（已截屏，但模型未返回有效描述。）"
+        return f"【当前屏幕视觉描述】\n{text.strip()}"
+
+class BrowserControlClass(BaseTool):
+    """统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL"""
+
+    args_schema: type[BaseModel] = create_model(
+        "BrowserControlInput",
+        action=(Literal["open", "close"], Field(
+            default="open",
+            description="操作类型：'open' 打开/刷新，'close' 关闭"
+        )),
+        url=(Optional[str], Field(
+            default=None,
+            description="URL 地址、'current' 表示当前页，留空则操作整个浏览器窗口"
+        )),
+        browser=(Optional[str], Field(
+            default=None,
+            description="浏览器名称，如 'edge', 'chrome', 'firefox'，不指定则使用系统默认"
+        )),
+    )
+    name: str = "browser_control"
+    description: str = (
+        "统一控制浏览器：打开/关闭浏览器、刷新/关闭标签页、打开指定 URL。\n"
+        "- action='open', url=None: 打开浏览器首页\n"
+        "- action='open', url='current': 刷新当前标签页\n"
+        "- action='open', url='https://...': 在新标签页打开指定 URL\n"
+        "- action='close', url=None: 关闭整个浏览器窗口\n"
+        "- action='close', url='current': 关闭当前标签页\n"
+        "- action='close', url='https://...': 关闭当前活动标签页"
+        "仅用于进行浏览器操作时调用，其余任何情况严禁调用。"
+    )
+
+    # ---------- 辅助方法 ----------
+    def _get_browser_info(self, browser_name: Optional[str]):
+        """获取浏览器控制器和窗口标题关键字"""
+        if browser_name is None:
+            return webbrowser.get(), None
+
+        name_lower = browser_name.lower().strip()
+        if name_lower not in BROWSER_MAP:
+            raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}")
+
+        reg_name, title_keyword = BROWSER_MAP[name_lower]
+        try:
+            controller = webbrowser.get(reg_name)
+        except webbrowser.Error:
+            controller = webbrowser.get(browser_name)
+        return controller, title_keyword
+
+    def _activate_window(self, title_keyword: str) -> bool:
+        """激活浏览器窗口（仅 Windows）"""
+        if sys.platform != "win32":
+            return False
+        try:
+            time.sleep(0.3)
+            windows = pyautogui.getWindowsWithTitle(title_keyword)
+            if windows:
+                win = windows[0]
+                if win.isMinimized:
+                    win.restore()
+                win.activate()
+                return True
+        except Exception:
+            pass
+        return False
+
+    def _send_keys(self, *keys):
+        """发送快捷键，根据平台调整"""
+        pyautogui.hotkey(*keys)
+
+    # ---------- 核心操作 ----------
+    def _open_action(self, url: Optional[str], browser: Optional[str]):
+        """执行打开类操作"""
+        controller, title_keyword = self._get_browser_info(browser)
+
+        if url is None:
+            # 仅打开浏览器首页（激活或启动）
+            controller.open("about:blank")
+            if title_keyword:
+                self._activate_window(title_keyword)
+            browser_info = f" (使用 {browser})" if browser else ""
+            return f"已打开浏览器首页{browser_info}"
+
+        elif url.lower() == "current":
+            # 刷新当前页
+            if title_keyword:
+                if not self._activate_window(title_keyword):
+                    return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新"
+            time.sleep(0.2)
+            if sys.platform == "darwin":
+                self._send_keys("command", "r")
+            else:
+                self._send_keys("f5")
+            return "已刷新当前标签页"
+
+        else:
+            # 打开指定 URL
+            controller.open(url)
+            if title_keyword:
+                self._activate_window(title_keyword)
+            return f"已尝试打开 URL: {url}"
+
+    def _close_action(self, url: Optional[str], browser: Optional[str]):
+        """执行关闭类操作"""
+        _, title_keyword = self._get_browser_info(browser)
+
+        if url is None:
+            # 关闭整个浏览器
+            if title_keyword:
+                if not self._activate_window(title_keyword):
+                    return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭"
+            time.sleep(0.2)
+            if sys.platform == "darwin":
+                self._send_keys("command", "q")
+            else:
+                self._send_keys("alt", "f4")
+            return "已尝试关闭浏览器窗口"
+
+        elif url.lower() == "current":
+            # 关闭当前标签页
+            if title_keyword:
+                if not self._activate_window(title_keyword):
+                    return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页"
+            time.sleep(0.2)
+            if sys.platform == "darwin":
+                self._send_keys("command", "w")
+            else:
+                self._send_keys("ctrl", "w")
+            return "已尝试关闭当前标签页"
+
+    def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str:
+        try:
+            if action == "open":
+                return self._open_action(url, browser)
+            elif action == "close":
+                return self._close_action(url, browser)
+            else:
+                return f"不支持的操作类型: {action}"
+        except Exception as e:
+            return f"执行浏览器操作时出错: {e}"
 
 if __name__ == "__main__":
-    # print(web_search.name)  # 输出工具名称
-    # print(web_search.description)  # 输出工具描述
-    # print(web_search.args_schema.model_json_schema())  # 输出工具参数结构
-    # # 测试工具
-    # query = "Python 中的列表和元组有什么区别？"
-    # result = web_search.invoke({"query": query})
-    # print(result)
-    print(date_query.name)  # 输出工具名称
-    print(date_query.description)  # 输出工具描述
-    print(date_query.args_schema.model_json_schema())  # 输出工具参数结构
-    print(date_query.invoke({}))  # 测试工具
-
+    # 这里可以进行一些简单的测试，例如创建工具实例并调用方法
+    pass

src/agent/tools/tool_main.py

@@ -1,252 +0,0 @@
-from __future__ import annotations
-
-import io
-from typing import Optional, Literal
-import sys
-import time
-import webbrowser
-import pyautogui
-
-from langchain.tools import BaseTool
-from pydantic import BaseModel, Field, create_model
-
-from agent.llm import Qwen36_35B_A3B_FP8_LLM
-from utils.env_utils import BROWSER_MAP
-
-
-class Websearchclass(BaseTool):
-        
-    args_schema : type[BaseModel] = create_model("Input_type", query=(str, Field(description="搜索查询内容")))
-    name : str = "web_search_class"
-    description : str = "一个网络搜索工具，可以根据用户的查询返回相关的搜索结果。仅当需要查询近期知识或已有知识无法解答需要联网查询时调用，其余任何情况严禁调用。"
-
-    def _get_search_wrapper(self):
-        from langchain_community.utilities import GoogleSerperAPIWrapper
-        SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5"
-        return GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048)
-
-    def _run(self, query: str) -> str:
-        try:
-            return self._get_search_wrapper().run(query)
-        except Exception as e:
-            return f"执行网络搜索时出错: {str(e)}"
-        
-    async def _arun(self, query: str) -> str:
-        try:
-            return await self._get_search_wrapper().arun(query)
-        except Exception as e:
-            return f"执行网络搜索时出错: {str(e)}"
-
-class DateQueryClass(BaseTool):
-
-    # 定义工具的输入参数结构，这里没有输入参数，所以是一个空的模型
-    args_schema : type[BaseModel] = create_model("NoneInput")
-    name : str = "date_query_class"
-    description : str = "一个日期查询工具，可以返回当前的日期信息。仅当需要查询当前时间时调用，其余任何情况严禁调用。"    
-
-    def _run(self) -> str:
-        try:
-            from datetime import datetime
-            now = datetime.now()
-            return now.strftime("%Y-%m-%d %H:%M:%S")
-        except Exception as e:
-            return f"执行日期查询时出错: {str(e)}"
-        
-    async def _arun(self) -> str:
-        return self._run()
-
-
-class ScreenCaptureDescribeClass(BaseTool):
-    """截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。"""
-
-    args_schema: type[BaseModel] = create_model(
-        "ScreenCaptureInput",
-        focus_query=(
-            Optional[str],
-            Field(
-                default=None,
-                description="可选。希望重点观察的界面或问题，例如「错误提示全文」「浏览器地址栏」；留空则整体描述屏幕。",
-            ),
-        ),
-    )
-    name: str = "screen_capture_describe"
-    description: str = (
-        "截取当前主屏幕画面，并用视觉模型转为文字描述。"
-        "注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。"
-    )
-
-    def _run(self, focus_query: Optional[str] = None) -> str:
-        # 1. 确保 pyautogui 可用
-        try:
-            import pyautogui
-        except ImportError as e:
-            return (
-                f"未安装 pyautogui，无法截屏: {e}。"
-                "请在项目环境中执行: pip install pyautogui"
-            )
-
-        # 2. 截屏并转为 PNG 字节流
-        try:
-            pil_img = pyautogui.screenshot()
-            buf = io.BytesIO()
-            pil_img.save(buf, format="PNG")
-            raw_png = buf.getvalue()
-        except Exception as e:
-            return f"截屏失败: {e}"
-
-        # 3. 构建描述指令
-        query = (focus_query or "").strip()
-        
-        # 4. 调用与对话图片上传完全相同的视觉模型进行描述
-        try:
-            # Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义
-            text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query)
-        except Exception as e:
-            return f"屏幕图像视觉解析失败: {e}"
-
-        if not (text or "").strip():
-            return "（已截屏，但模型未返回有效描述。）"
-        return f"【当前屏幕视觉描述】\n{text.strip()}"
-
-class BrowserControlClass(BaseTool):
-    """统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL"""
-
-    args_schema: type[BaseModel] = create_model(
-        "BrowserControlInput",
-        action=(Literal["open", "close"], Field(
-            default="open",
-            description="操作类型：'open' 打开/刷新，'close' 关闭"
-        )),
-        url=(Optional[str], Field(
-            default=None,
-            description="URL 地址、'current' 表示当前页，留空则操作整个浏览器窗口"
-        )),
-        browser=(Optional[str], Field(
-            default=None,
-            description="浏览器名称，如 'edge', 'chrome', 'firefox'，不指定则使用系统默认"
-        )),
-    )
-    name: str = "browser_control"
-    description: str = (
-        "统一控制浏览器：打开/关闭浏览器、刷新/关闭标签页、打开指定 URL。\n"
-        "- action='open', url=None: 打开浏览器首页\n"
-        "- action='open', url='current': 刷新当前标签页\n"
-        "- action='open', url='https://...': 在新标签页打开指定 URL\n"
-        "- action='close', url=None: 关闭整个浏览器窗口\n"
-        "- action='close', url='current': 关闭当前标签页\n"
-        "- action='close', url='https://...': 关闭当前活动标签页"
-        "仅用于进行浏览器操作时调用，其余任何情况严禁调用。"
-    )
-
-    # ---------- 辅助方法 ----------
-    def _get_browser_info(self, browser_name: Optional[str]):
-        """获取浏览器控制器和窗口标题关键字"""
-        if browser_name is None:
-            return webbrowser.get(), None
-
-        name_lower = browser_name.lower().strip()
-        if name_lower not in BROWSER_MAP:
-            raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}")
-
-        reg_name, title_keyword = BROWSER_MAP[name_lower]
-        try:
-            controller = webbrowser.get(reg_name)
-        except webbrowser.Error:
-            controller = webbrowser.get(browser_name)
-        return controller, title_keyword
-
-    def _activate_window(self, title_keyword: str) -> bool:
-        """激活浏览器窗口（仅 Windows）"""
-        if sys.platform != "win32":
-            return False
-        try:
-            time.sleep(0.3)
-            windows = pyautogui.getWindowsWithTitle(title_keyword)
-            if windows:
-                win = windows[0]
-                if win.isMinimized:
-                    win.restore()
-                win.activate()
-                return True
-        except Exception:
-            pass
-        return False
-
-    def _send_keys(self, *keys):
-        """发送快捷键，根据平台调整"""
-        pyautogui.hotkey(*keys)
-
-    # ---------- 核心操作 ----------
-    def _open_action(self, url: Optional[str], browser: Optional[str]):
-        """执行打开类操作"""
-        controller, title_keyword = self._get_browser_info(browser)
-
-        if url is None:
-            # 仅打开浏览器首页（激活或启动）
-            controller.open("about:blank")
-            if title_keyword:
-                self._activate_window(title_keyword)
-            browser_info = f" (使用 {browser})" if browser else ""
-            return f"已打开浏览器首页{browser_info}"
-
-        elif url.lower() == "current":
-            # 刷新当前页
-            if title_keyword:
-                if not self._activate_window(title_keyword):
-                    return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新"
-            time.sleep(0.2)
-            if sys.platform == "darwin":
-                self._send_keys("command", "r")
-            else:
-                self._send_keys("f5")
-            return "已刷新当前标签页"
-
-        else:
-            # 打开指定 URL
-            controller.open(url)
-            if title_keyword:
-                self._activate_window(title_keyword)
-            return f"已尝试打开 URL: {url}"
-
-    def _close_action(self, url: Optional[str], browser: Optional[str]):
-        """执行关闭类操作"""
-        _, title_keyword = self._get_browser_info(browser)
-
-        if url is None:
-            # 关闭整个浏览器
-            if title_keyword:
-                if not self._activate_window(title_keyword):
-                    return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭"
-            time.sleep(0.2)
-            if sys.platform == "darwin":
-                self._send_keys("command", "q")
-            else:
-                self._send_keys("alt", "f4")
-            return "已尝试关闭浏览器窗口"
-
-        elif url.lower() == "current":
-            # 关闭当前标签页
-            if title_keyword:
-                if not self._activate_window(title_keyword):
-                    return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页"
-            time.sleep(0.2)
-            if sys.platform == "darwin":
-                self._send_keys("command", "w")
-            else:
-                self._send_keys("ctrl", "w")
-            return "已尝试关闭当前标签页"
-
-    def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str:
-        try:
-            if action == "open":
-                return self._open_action(url, browser)
-            elif action == "close":
-                return self._close_action(url, browser)
-            else:
-                return f"不支持的操作类型: {action}"
-        except Exception as e:
-            return f"执行浏览器操作时出错: {e}"
-
-if __name__ == "__main__":
-    # 这里可以进行一些简单的测试，例如创建工具实例并调用方法
-    pass

src/agent/tools/tool_rag.py

@@ -0,0 +1,373 @@
+# agent/tools/tool_rag.py
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+import time
+from datetime import datetime
+from typing import List, Optional, Union, Literal
+
+import chromadb
+import requests
+from langchain.tools import BaseTool
+from pydantic import BaseModel, Field, create_model
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from utils.env_utils import RAG_BASE_URL, RAG_CHUNK_SIZE, RAG_CHUNK_OVERLAP
+
+# ---------- 新增 RAG 工具 ----------
+# Chroma 持久化路径：agent/memory/chroma_rag_db
+CHROMA_PERSIST_DIR = os.path.join(os.path.dirname(__file__), '..', 'memory', 'chroma_rag_db')
+
+# 全局 Chroma 客户端和集合缓存
+_chroma_client = None
+_collection_cache = {}
+
+def get_chroma_client():
+    global _chroma_client
+    if _chroma_client is None:
+        _chroma_client = chromadb.PersistentClient(path=CHROMA_PERSIST_DIR)
+    return _chroma_client
+
+def get_collection(agent_name: str):
+    """获取或创建某个 agent 的 Collection"""
+    client = get_chroma_client()
+    collection_name = f"rag_{agent_name}"
+    if collection_name not in _collection_cache:
+        # 若 Collection 不存在则创建，使用余弦相似度
+        try:
+            collection = client.get_collection(name=collection_name, embedding_function=None)
+        except Exception:
+            collection = client.create_collection(
+                name=collection_name,
+                metadata={"hnsw:space": "cosine"}
+            )
+        _collection_cache[collection_name] = collection
+    return _collection_cache[collection_name]
+
+# ---------- Embedding 接口封装 ----------
+def embed_texts(queries: List[str], documents: List[str] = None) -> tuple:
+    """
+    调用 /embed，返回 (query_embeddings, doc_embeddings)
+    """
+    payload = {
+        "task_description": "Given a web search query, retrieve relevant passages that answer the query",
+        "queries": queries,
+        "documents": documents or []
+    }
+    try:
+        resp = requests.post(f"{RAG_BASE_URL}/embed", json=payload, timeout=30)
+        resp.raise_for_status()
+        data = resp.json()
+        return data["query_embeddings"], data.get("doc_embeddings", [])
+    except Exception as e:
+        raise RuntimeError(f"Embedding 请求失败: {e}")
+
+def rerank_documents(query: str, documents: List[str]) -> List[float]:
+    """调用 /rerank，返回排序后的分数列表"""
+    payload = {
+        "task_description": "Given a web search query, retrieve relevant passages that answer the query",
+        "query": query,
+        "documents": documents
+    }
+    try:
+        resp = requests.post(f"{RAG_BASE_URL}/rerank", json=payload, timeout=30)
+        resp.raise_for_status()
+        return resp.json()["scores"]
+    except Exception as e:
+        raise RuntimeError(f"Reranker 请求失败: {e}")
+
+# ---------- 文本分块 ----------
+def split_into_chunks(text: str) -> List[str]:
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=CHUNK_SIZE,
+        chunk_overlap=CHUNK_OVERLAP,
+        separators=["\n\n", "\n", " ", ""]
+    )
+    return splitter.split_text(text)
+
+# ---------- 文件解析 ----------
+def parse_file_to_chunks(file_path: str) -> List[dict]:
+    """
+    解析文件，返回每个 chunk 的字典，包含：
+    - content: 文本内容
+    - source: 详细来源描述
+    """
+    ext = os.path.splitext(file_path)[1].lower()
+    chunks = []
+
+    if ext == ".txt" or ext == ".md":
+        with open(file_path, "r", encoding="utf-8") as f:
+            text = f.read()
+        texts = split_into_chunks(text)
+        for i, t in enumerate(texts):
+            chunks.append({
+                "content": t,
+                "source": f"{os.path.basename(file_path)} (chunk {i+1}/{len(texts)})"
+            })
+
+    elif ext == ".json":
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        text = json.dumps(data, ensure_ascii=False, indent=2)
+        texts = split_into_chunks(text)
+        for i, t in enumerate(texts):
+            chunks.append({
+                "content": t,
+                "source": f"{os.path.basename(file_path)} (JSON, chunk {i+1}/{len(texts)})"
+            })
+
+    elif ext == ".csv":
+        import pandas as pd
+        df = pd.read_csv(file_path)
+        text = df.to_string(index=False)
+        texts = split_into_chunks(text)
+        for i, t in enumerate(texts):
+            chunks.append({
+                "content": t,
+                "source": f"{os.path.basename(file_path)} (CSV, chunk {i+1}/{len(texts)})"
+            })
+
+    elif ext == ".docx":
+        from docx import Document
+        doc = Document(file_path)
+        full_text = "\n".join([p.text for p in doc.paragraphs])
+        texts = split_into_chunks(full_text)
+        for i, t in enumerate(texts):
+            chunks.append({
+                "content": t,
+                "source": f"{os.path.basename(file_path)} (DOCX, chunk {i+1}/{len(texts)})"
+            })
+
+    elif ext == ".pdf":
+        import pdfplumber
+        with pdfplumber.open(file_path) as pdf:
+            for page_num, page in enumerate(pdf.pages, start=1):
+                page_text = page.extract_text() or ""
+                if page_text.strip():
+                    page_chunks = split_into_chunks(page_text)
+                    for i, t in enumerate(page_chunks):
+                        chunks.append({
+                            "content": t,
+                            "source": f"{os.path.basename(file_path)} (Page {page_num}, chunk {i+1})"
+                        })
+
+    else:
+        raise ValueError(f"不支持的文件格式: {ext}")
+
+    return chunks
+
+# ---------- RAG 工具本体 ----------
+class RAGTool(BaseTool):
+    name: str = "rag_tool"
+    description: str = (
+        "通用知识库工具，支持增删查操作。\n"
+        "参数说明：\n"
+        "- agent_name: 所属 Agent 名称或名称列表，必填。\n"
+        "- action_type: 操作类型，必填。可选：'query'(查询), 'add_document'(添加文档), 'clear'(清除)。\n"
+        "- content: 查询内容(用于query) 或 直接文本内容(用于add_document，与source二选一)。\n"
+        "- source: 文件路径(用于add_document，与content二选一)。\n"
+        "- filter: 过滤条件(用于query或clear)。\n"
+        "使用示例：\n"
+        "- 查询：action_type='query', content='用户问题', agent_name='main'\n"
+        "- 添加文本：action_type='add_document', content='知识文本', agent_name='main'\n"
+        "- 添加文件：action_type='add_document', source='/path/to/file.pdf', agent_name=['agent1','agent2']\n"
+        "- 清除：action_type='clear', filter='某个来源', agent_name='main'"
+    )
+
+    args_schema: type[BaseModel] = create_model(
+        "RAGInput",
+        agent_name=(Union[str, List[str]], Field(..., description="所属 Agent 名称或名称列表")),
+        action_type=(Literal["query", "add_document", "clear"], Field(..., description="操作类型")),
+        content=(Optional[str], Field(None, description="查询内容或直接添加的文本内容")),
+        source=(Optional[str], Field(None, description="文件路径，用于添加文档")),
+        filter=(Optional[Union[str, List[str]]], Field(None, description="过滤条件，用于查询或清除"))
+    )
+
+    def _run(
+        self,
+        agent_name: Union[str, List[str]],
+        action_type: str,
+        content: Optional[str] = None,
+        source: Optional[str] = None,
+        filter: Optional[Union[str, List[str]]] = None
+    ) -> str:
+        try:
+            # 标准化 agent_name 为列表
+            agents = [agent_name] if isinstance(agent_name, str) else agent_name
+
+            if action_type == "query":
+                return self._handle_query(agents, content, filter)
+            elif action_type == "add_document":
+                return self._handle_add(agents, content, source)
+            elif action_type == "clear":
+                return self._handle_clear(agents, filter)
+            else:
+                return f"未知操作: {action_type}"
+        except Exception as e:
+            return f"RAG 工具执行出错: {str(e)}"
+
+    # ---------- 查询 ----------
+    def _handle_query(self, agents: List[str], query: str, filter: Optional[Union[str, List[str]]]) -> str:
+        if not query:
+            return "查询内容不能为空。"
+
+        # 1. 获取查询向量
+        q_embs, _ = embed_texts(queries=[query])
+        query_vector = q_embs[0]
+
+        # 2. 从相关 Collection 中检索
+        all_results = []
+        for agent in agents:
+            coll = get_collection(agent)
+            # 构建 where 条件（基础过滤可叠加元数据过滤，这里简单处理）
+            where = None
+            if filter:
+                if isinstance(filter, str):
+                    # 简单实现：按 source 字段包含 filter 字符串
+                    where = {"source": {"$contains": filter}}
+                else:
+                    # 列表情况，暂不支持，返回空
+                    pass
+
+            res = coll.query(
+                query_embeddings=[query_vector],
+                n_results=10,
+                where=where
+            )
+            # 整理结果：[(id, document, metadata), ...]
+            if res["ids"] and res["ids"][0]:
+                ids = res["ids"][0]
+                docs = res["documents"][0]
+                metas = res["metadatas"][0]
+                for i, d, m in zip(ids, docs, metas):
+                    all_results.append({"id": i, "document": d, "metadata": m})
+
+        if not all_results:
+            return "未在知识库中找到相关内容。"
+
+        # 去重（同一个文档可能被多个 agent 同时索引）
+        seen = set()
+        unique_results = []
+        for r in all_results:
+            if r["document"] not in seen:
+                seen.add(r["document"])
+                unique_results.append(r)
+        unique_results = unique_results[:10]  # 确保最多10条
+
+        # 3. Reranker 精排
+        docs_for_rerank = [r["document"] for r in unique_results]
+        scores = rerank_documents(query, docs_for_rerank)
+
+        # 按分数降序取前3
+        sorted_items = sorted(
+            zip(unique_results, scores),
+            key=lambda x: x[1],
+            reverse=True
+        )[:3]
+
+        if not sorted_items:
+            return "Reranker 未返回有效结果。"
+
+        # 拼接输出
+        outputs = []
+        for item, score in sorted_items:
+            meta = item["metadata"]
+            source_info = meta.get("source", "未知来源")
+            outputs.append(f"【来源：{source_info}（相关度 {score:.3f}）】\n{item['document']}")
+
+        return "\n\n".join(outputs)
+
+    # ---------- 添加文档 ----------
+    def _handle_add(self, agents: List[str], content: Optional[str], source: Optional[str]) -> str:
+        if not content and not source:
+            return "必须提供 content 或 source。"
+        if content and source:
+            return "content 和 source 只能提供一个。"
+
+        # 1. 准备 chunks 列表
+        chunks_meta = []  # 元素为 (text, source_desc)
+        timestamp = datetime.now().isoformat()
+
+        if content:
+            # 直接使用 content 分块
+            texts = split_into_chunks(content)
+            chunks_meta = [(t, "直接输入") for t in texts]
+
+        elif source:
+            # 解析文件
+            if not os.path.isfile(source):
+                return f"文件不存在: {source}"
+            chunks_meta = [(c["content"], c["source"]) for c in parse_file_to_chunks(source)]
+
+        if not chunks_meta:
+            return "文档内容为空，未添加任何条目。"
+
+        # 2. 批量获取文档向量
+        doc_texts = [cm[0] for cm in chunks_meta]
+        _, doc_embs = embed_texts(queries=["dummy"], documents=doc_texts)
+
+        # 3. 为每个 agent 写入
+        total_added = 0
+        for agent in agents:
+            coll = get_collection(agent)
+            ids = []
+            embeddings = []
+            metadatas = []
+            documents = []
+
+            for idx, (text, src_desc) in enumerate(chunks_meta):
+                # 使用 hash 作为唯一 ID
+                uid = hashlib.md5(f"{agent}_{timestamp}_{text[:80]}".encode()).hexdigest()
+                ids.append(uid)
+                embeddings.append(doc_embs[idx])
+                metadatas.append({
+                    "source": src_desc,
+                    "agent_name": agent,
+                    "timestamp": timestamp,
+                })
+                documents.append(text)
+
+            coll.add(
+                ids=ids,
+                embeddings=embeddings,
+                metadatas=metadatas,
+                documents=documents
+            )
+            total_added += len(ids)
+
+        return f"成功添加 {total_added} 条知识块到 {', '.join(agents)}。"
+
+    # ---------- 清除 ----------
+    def _handle_clear(self, agents: List[str], filter: Optional[Union[str, List[str]]]) -> str:
+        if not filter:
+            return "清除操作必须提供 filter 条件。"
+
+        total_deleted = 0
+        for agent in agents:
+            coll = get_collection(agent)
+            # 根据 filter 构建 where，当前简化：假设 filter 是 source 的包含条件
+            where = None
+            if isinstance(filter, str):
+                where = {"source": {"$contains": filter}}
+            elif isinstance(filter, list):
+                # 多个条件用 $or
+                where = {"$or": [{"source": {"$contains": f}} for f in filter]}
+
+            try:
+                # 查询要删除的 IDs
+                res = coll.get(where=where)
+                if res["ids"]:
+                    coll.delete(ids=res["ids"])
+                    total_deleted += len(res["ids"])
+            except Exception as e:
+                return f"清除时出错 (agent={agent}): {e}"
+
+        return f"已从 {', '.join(agents)} 中删除 {total_deleted} 条记录。"
+
+    async def _arun(self, **kwargs):
+        # 直接调用同步版本
+        return self._run(**kwargs)
+        

src/agent/utils/agent_utils.py

@@ -1,15 +1,23 @@
-# agent/agent_utils.py
+# agent/utils/agent_utils.py
 import json
 import os
 import shutil
+import sqlite3
+from typing import List, Tuple, Any
 
 from langchain_community.chat_message_histories import SQLChatMessageHistory
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.messages import BaseMessage, SystemMessage, HumanMessage
+from langchain_core.prompts import ChatPromptTemplate
 
 from agent.memory.system_prompt import SUMMARY_SYSTEM_PROMPT
+from agent.utils.env_utils import (
+    LLM_CONTEXT_WINDOW,
+    MINI_COMPRESS_RATE,
+    HARD_COMPRESS_RATE,
+    CURRENT_TURNS,
+)
 
-# 关键改动：__file__ 在 agent/utils/agent_utils.py 下，
-# 取两次 dirname 得到 agent 目录（即 src/agent）
+# ---------- 目录与数据库路径 ----------
 _AGENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 HISTORY_ROOT = os.path.join(_AGENT_DIR, "history")
 _DB_DIR = os.path.join(HISTORY_ROOT, "data_base")
@@ -18,58 +26,234 @@ MULTI_INPUTS_ROOT = os.path.join(HISTORY_ROOT, "multi_inputs")
 
 def ensure_dirs_and_migrate_db() -> None:
     os.makedirs(_DB_DIR, exist_ok=True)
-    os.makedirs(os.path.join(HISTORY_ROOT, "multi_inputs"), exist_ok=True)
+    os.makedirs(MULTI_INPUTS_ROOT, exist_ok=True)
+    # 迁移旧数据库
     old_db = os.path.join(HISTORY_ROOT, "chat_history.db")
     if os.path.isfile(old_db) and not os.path.isfile(CHAT_HISTORY_DB_FS):
         shutil.move(old_db, CHAT_HISTORY_DB_FS)
 
+    conn = sqlite3.connect(CHAT_HISTORY_DB_FS)
+    # 创建 message_store 表（与 SQLChatMessageHistory 一致）
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS message_store (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT NOT NULL,
+            message TEXT NOT NULL
+        )
+    """)
+    # 创建 session_meta 表（我们的元数据表）
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS session_meta (
+            session_id TEXT NOT NULL,
+            agent_name TEXT NOT NULL,
+            cursor_index INTEGER DEFAULT -1,
+            compressed_content TEXT DEFAULT '',
+            last_total_tokens INTEGER DEFAULT 0,
+            PRIMARY KEY (session_id, agent_name)
+        )
+    """)
+    conn.commit()
+    conn.close()
+
 ensure_dirs_and_migrate_db()
 
+# ---------- 历史记录操作 ----------
 def get_history(session_id: str) -> SQLChatMessageHistory:
     return SQLChatMessageHistory(
         session_id=session_id,
         connection="sqlite:///" + CHAT_HISTORY_DB_FS.replace("\\", "/"),
     )
 
-def summarize_history(query, config):
-    """
-    剪辑最近8条消息，并对更早的消息生成摘要。
-    返回 {'summary': str|None, 'chat_history': list[BaseMessage]}
-    """
-    session_id = config["configurable"]["session_id"]
-    if not session_id:
-        raise ValueError("Session ID is required in the config.")
-    chat_history = get_history(session_id)
-    stored_messages = chat_history.messages
-
-    if len(stored_messages) <= 8:
-        return {"summary": None, "chat_history": stored_messages}
-
-    recent = stored_messages[-8:]
-    old = stored_messages[:-8]
-    
-    summary_prompt = ChatPromptTemplate.from_messages([
-        ("system", SUMMARY_SYSTEM_PROMPT),
-        MessagesPlaceholder(variable_name="messages_to_summarize", optional=True),
-        ("human", "请生成包含上述对话核心内容的摘要，保留重要事实和决策。"),
-    ])
-
-    llm = config.get("llm")
-    chain = summary_prompt | llm
-    summary = chain.invoke(
-        {"messages_to_summarize": old},
-        config={
-            "metadata": {
-                "output_type": "text",
-                "audio_output_dir": None,
-            }
-        }
+# ---------- session_meta 元数据管理 ----------
+def get_session_meta(session_id: str, agent_name: str) -> dict:
+    conn = sqlite3.connect(CHAT_HISTORY_DB_FS)
+    cur = conn.execute(
+        "SELECT cursor_index, compressed_content, last_total_tokens FROM session_meta WHERE session_id=? AND agent_name=?",
+        (session_id, agent_name)
     )
-    return {"summary": summary.content, "chat_history": recent}
+    row = cur.fetchone()
+    conn.close()
+    if row:
+        return {
+            "cursor_index": row[0],
+            "compressed_content": row[1],
+            "last_total_tokens": row[2]
+        }
+    return {"cursor_index": -1, "compressed_content": "", "last_total_tokens": 0}
 
+def update_session_meta(session_id: str, agent_name: str,
+                        cursor_index: int, compressed_content: str, last_total_tokens: int) -> None:
+    conn = sqlite3.connect(CHAT_HISTORY_DB_FS)
+    conn.execute("""
+        INSERT INTO session_meta (session_id, agent_name, cursor_index, compressed_content, last_total_tokens)
+        VALUES (?, ?, ?, ?, ?)
+        ON CONFLICT(session_id, agent_name) DO UPDATE SET
+            cursor_index=excluded.cursor_index,
+            compressed_content=excluded.compressed_content,
+            last_total_tokens=excluded.last_total_tokens
+    """, (session_id, agent_name, cursor_index, compressed_content, last_total_tokens))
+    conn.commit()
+    conn.close()
 
+# ---------- 工具函数 ----------
+def _get_messages_since_cursor(history: SQLChatMessageHistory, cursor_index: int) -> List[BaseMessage]:
+    all_msgs = history.messages
+    if cursor_index < 0 or cursor_index >= len(all_msgs) - 1:
+        return all_msgs
+    return all_msgs[cursor_index + 1:]
+
+def _count_turns(messages: List[BaseMessage]) -> int:
+    turns = 0
+    i = 0
+    n = len(messages)
+    while i < n:
+        if messages[i].type == "human":
+            if i + 1 < n and messages[i + 1].type in ("ai", "assistant"):
+                turns += 1
+                i += 2
+            else:
+                i += 1
+        else:
+            i += 1
+    return turns
+
+def _last_turn_end_index(messages: List[BaseMessage]) -> int:
+    for i in range(len(messages) - 1, -1, -1):
+        if messages[i].type in ("ai", "assistant"):
+            return i
+    return -1
+
+# ---------- 压缩函数 ----------
+def _lightweight_compress(llm, messages_to_compress: List[BaseMessage]) -> str:
+    """轻量压缩：保留关键事实、实体、决策，删除冗余"""
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", SUMMARY_SYSTEM_PROMPT),
+        ("human", "请对以下对话进行轻量压缩，保留核心事实、实体、决策和对话流向。\n\n{chat}"),
+    ])
+    chain = prompt | llm
+    chat_text = "\n".join([
+        f"{'用户' if m.type == 'human' else '助手'}: {m.content}" for m in messages_to_compress
+    ])
+    print("轻量化压缩函数被调用")
+    print("需要压缩的内容为"+chat_text)
+    result = chain.invoke({"chat": chat_text}, config={"metadata": {"output_type": "text","audio_output_dir": None,}})
+    return result.content
+
+def _hard_compress(llm, compressed_existing: str, new_chat_text: str) -> str:
+    """全量压缩：将已有压缩摘要和新对话内容融合为最终摘要"""
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", SUMMARY_SYSTEM_PROMPT),
+        ("human", "这是之前的对话摘要：\n{previous_summary}\n\n以及后续对话内容：\n{new_chat}\n\n请生成包含所有信息的最终摘要，保留重要事实和决策。"),
+    ])
+    chain = prompt | llm
+    print("全量压缩函数被调用")
+    result = chain.invoke({
+        "previous_summary": compressed_existing,
+        "new_chat": new_chat_text
+    }, config={"metadata": {"output_type": "text"}})
+    return result.content
+
+# ---------- 核心：上下文感知构建器 (summarizer) ----------
+def summarizer(
+    session_id: str,
+    agent_name: str,
+    system_prompt: str,
+    user_content: str,
+    llm: Any
+) -> Tuple[List[BaseMessage], bool]:
+    """
+    根据上下文窗口与上一轮 token 使用量，动态压缩历史消息并返回完整消息列表。
+    :param session_id: 会话ID
+    :param agent_name: Agent 名称
+    :param system_prompt: 系统提示词
+    :param user_content: 当前用户输入文本（纯文本或已处理的多模态文本）
+    :param llm: 语言模型实例（用于压缩）
+    :return: (messages, context_exceeded)
+        messages: 构建好的消息列表
+        context_exceeded: 是否超过 90% 窗口，需要提示新建会话
+    """
+    meta = get_session_meta(session_id, agent_name)
+    cursor_index = meta["cursor_index"]
+    compressed_content = meta["compressed_content"]
+    last_total_tokens = meta["last_total_tokens"]
+
+    # 检查上下文是否已满
+    if last_total_tokens > LLM_CONTEXT_WINDOW * 0.9:
+        return [], True
+
+    # 判断是否需要触发压缩
+    need_light = False
+    need_hard = False
+    if last_total_tokens > 0:
+        if last_total_tokens > LLM_CONTEXT_WINDOW * HARD_COMPRESS_RATE:
+            need_hard = True
+        elif last_total_tokens > LLM_CONTEXT_WINDOW * MINI_COMPRESS_RATE:
+            need_light = True
+
+    history = get_history(session_id)
+    all_messages = history.messages
+
+    if need_light:
+        # 轻量压缩：保留最近 CURRENT_TURNS 轮对话不压缩
+        turns = _count_turns(all_messages)
+        keep_turns = min(CURRENT_TURNS, turns)
+        # 找到需要保留的轮数之后的最后一条消息索引
+        idx = len(all_messages) - 1
+        turn_count = 0
+        while idx >= 0:
+            if all_messages[idx].type in ("ai", "assistant"):
+                turn_count += 1
+                if turn_count == keep_turns:
+                    break
+            idx -= 1
+        new_cursor = idx - 1 if idx >= 0 else -1   # idx 是保留轮次开始的前一条消息索引
+
+        # 待压缩内容 = 游标及之前的所有消息
+        to_compress = []
+        if new_cursor >= 0:
+            to_compress = all_messages[:new_cursor + 1]
+        if compressed_content:
+            to_compress.insert(0, HumanMessage(content=f"[先前摘要] {compressed_content}"))
+
+        new_compressed = _lightweight_compress(llm, to_compress) if to_compress else compressed_content
+        # 更新游标和压缩内容
+        update_session_meta(session_id, agent_name, new_cursor, new_compressed, last_total_tokens)
+        meta["cursor_index"] = new_cursor
+        meta["compressed_content"] = new_compressed
+
+    elif need_hard:
+        # 全量压缩：将已有压缩内容 + 游标后的所有消息合并为最终摘要
+        after_cursor = _get_messages_since_cursor(history, cursor_index)
+        new_chat_text = "\n".join(
+            [f"{'用户' if m.type == 'human' else '助手'}: {m.content}" for m in after_cursor]
+        )
+        if compressed_content:
+            new_compressed = _hard_compress(llm, compressed_content, new_chat_text)
+        else:
+            new_compressed = _hard_compress(llm, "", new_chat_text)
+        # 游标移到消息列表末尾（全量压缩后无未压缩消息）
+        last_msg_idx = len(all_messages) - 1 if all_messages else -1
+        update_session_meta(session_id, agent_name, last_msg_idx, new_compressed, last_total_tokens)
+        meta["cursor_index"] = last_msg_idx
+        meta["compressed_content"] = new_compressed
+
+    # 构建返回消息列表
+    system_text = system_prompt
+    if meta["compressed_content"]:
+        system_text += f"\n\n【历史对话摘要】\n{meta['compressed_content']}"
+
+    messages = [SystemMessage(content=system_text)]
+    # 添加游标之后的未压缩消息
+    recent = _get_messages_since_cursor(history, meta["cursor_index"])
+    messages.extend(recent)
+    # 添加当前用户消息
+    messages.append(HumanMessage(content=user_content))
+
+    return messages, False
+
+# ---------- 辅助函数 ----------
 def assistant_text(value) -> str:
-    """把网关返回的 content、reasoning 等（str 或块列表）收成一段可见文本。"""
+    """把网关返回的 content、reasoning 等收成一段可见文本。"""
     if value is None:
         return ""
     if isinstance(value, str):
@@ -95,87 +279,4 @@ def assistant_text(value) -> str:
             return json.dumps(value, ensure_ascii=False)
         except TypeError:
             return str(value).strip()
-    return str(value).strip()
-
-
-def normalize_user_text_content(value) -> str:
-    """尽量把用户输入收敛为纯文本，避免把 dict/list 结构直接送进主模型。"""
-    if value is None:
-        return ""
-    if isinstance(value, str):
-        return value.strip()
-    if isinstance(value, dict):
-        for key in ("text", "content", "input", "query", "question", "message"):
-            v = value.get(key)
-            if isinstance(v, str) and v.strip():
-                return v.strip()
-        content = value.get("content")
-        if content is not None:
-            text = normalize_user_text_content(content)
-            if text:
-                return text
-        return assistant_text(value)
-    if isinstance(value, list):
-        parts: list[str] = []
-        for item in value:
-            if isinstance(item, str) and item.strip():
-                parts.append(item.strip())
-            elif isinstance(item, dict):
-                if item.get("type") == "text":
-                    text = (item.get("text") or "").strip()
-                    if text:
-                        parts.append(text)
-                else:
-                    text = normalize_user_text_content(item)
-                    if text:
-                        parts.append(text)
-        return "\n".join(parts).strip()
-    return str(value).strip()
-
-
-def extract_result_from_invoke(value) -> str:
-    """兼容不同 invoke 返回结构，尽量提取可显示的助手文本。"""
-    if value is None:
-        return ""
-
-    if hasattr(value, "content"):
-        text = assistant_text(getattr(value, "content", None))
-        if text:
-            return text
-
-    if isinstance(value, dict):
-        for key in ("output", "content", "text", "answer", "result"):
-            text = assistant_text(value.get(key))
-            if text:
-                return text
-
-        message = value.get("message")
-        if message is not None:
-            text = extract_result_from_invoke(message)
-            if text:
-                return text
-
-        generations = value.get("generations")
-        if isinstance(generations, list):
-            for generation in generations:
-                text = extract_result_from_invoke(generation)
-                if text:
-                    return text
-
-    if isinstance(value, list):
-        for item in value:
-            text = extract_result_from_invoke(item)
-            if text:
-                return text
-
-    if hasattr(value, "message"):
-        text = extract_result_from_invoke(getattr(value, "message", None))
-        if text:
-            return text
-
-    if hasattr(value, "additional_kwargs"):
-        text = assistant_text(getattr(value, "additional_kwargs", {}).get("reasoning_content"))
-        if text:
-            return text
-
-    return ""
+    return str(value).strip()

src/agent/utils/env_utils.py

@@ -28,6 +28,13 @@ LLM_TIMEOUT = _get_env("LLM_TIMEOUT", default="60", cast=float)
 TTS_BASE_URL = _get_env("TTS_BASE_URL")
 ASR_BASE_URL = _get_env("ASR_BASE_URL")
 RAG_BASE_URL = _get_env("RAG_BASE_URL")
+LLM_CONTEXT_WINDOW = int(os.getenv("LLM_CONTEXT_WINDOW", 131072))
+MINI_COMPRESS_RATE = float(os.getenv("MINI_COMPRESS_RATE", 0.5))
+HARD_COMPRESS_RATE = float(os.getenv("HARD_COMPRESS_RATE", 0.75))
+CURRENT_TURNS = int(os.getenv("CURRENT_TURNS", 4))
+IMG_SIZE = int(os.getenv("IMG_SIZE", 768))
+RAG_CHUNK_SIZE=int(os.getenv("RAG_CHUNK_SIZE", 500))
+RAG_CHUNK_OVERLAP=int(os.getenv("RAG_CHUNK_OVERLAP", 50))
 
 BROWSER_MAP: Dict[str, Tuple[str, str]] = {
     "edge": ("microsoft-edge", "Microsoft Edge"),

src/agent/utils/image_utils.py

@@ -1,13 +1,11 @@
 """图片缩放、体积控制与 OpenAI 风格多模态 image_url 片段（供 LLM / Agent 使用）。"""
-from __future__ import annotations
-
 import base64
 import io
 import os
 from typing import Any, Protocol
-
 from PIL import Image
 
+from agent.utils.env_utils import IMG_SIZE
 
 class SupportsChatCompletion(Protocol):
     """具备 ``chat_completion(messages, **kwargs)`` 的 LLM（如 ``ChatQwen35OpenAI``）。"""
@@ -26,7 +24,7 @@ def guess_image_mime(extension_without_dot: str) -> str:
     return "image/jpeg"
 
 
-def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = 768) -> bytes:
+def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = IMG_SIZE) -> bytes:
     """
     控制图片体积，避免 data-url base64 触发超长 token。
     - 若图片尺寸过大，等比例缩放（默认最大边 max_size）
@@ -56,9 +54,6 @@ def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int =
         new_width = int(original_width * ratio)
         new_height = int(original_height * ratio)
         img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
-        print(f"图片已从 {original_width}x{original_height} 缩放到 {new_width}x{new_height}")
-    else:
-        print(f"图片尺寸 {original_width}x{original_height} 未超过限制，无需缩放")
 
     img_buffer = io.BytesIO()
     save_format = (original_format or "PNG").upper()
@@ -87,7 +82,6 @@ def image_to_base64(image_content: str | bytes) -> str | None:
         processed = resize_image_if_needed(image_content)
         return base64.b64encode(processed).decode("utf-8")
     except Exception as e:
-        print(f"处理图片 {image_content!r} 时出错: {e}")
         return None
 
 
@@ -111,7 +105,6 @@ def image_bytes_to_openai_image_url_part(
     try:
         processed = resize_image_if_needed(image_bytes)
     except Exception as e:
-        print(f"处理图片字节时出错: {e}")
         return None
     mime = _mime_from_magic(processed)
     b64_str = base64.b64encode(processed).decode("utf-8")

src/agent/web/app.css

@@ -6,7 +6,6 @@
   --radius: 14px;
   --sidebar-w: 272px;
   --rail-w: 44px;
-  /* 背景：景深模糊（px），由 JS 写入 html 行内或仍用默认 */
   --bg-dof: 0px;
   --bg-vignette: 0.35;
   --bubble-user: rgba(236, 254, 255, 0.55);
@@ -40,7 +39,7 @@ body {
   overflow-x: hidden;
 }
 
-/* 全屏背景 + 景深（对底图做轻微放大以裁掉模糊边缘） */
+/* 全屏背景 + 景深 */
 .bg-scene {
   position: fixed;
   inset: 0;
@@ -52,7 +51,6 @@ body {
 .bg-image {
   position: absolute;
   inset: 0;
-  /* 默认 cover；加载后 app.js 按 natural 尺寸与视口写死 px，等价 cover + 景深 bleed */
   background-color: #334155;
   background-image: url("/image/background.jpg");
   background-size: cover;
@@ -435,6 +433,7 @@ body {
   background: transparent;
 }
 
+/* ===== 消息容器 ===== */
 .msg {
   max-width: 95%;
   width: fit-content;
@@ -444,19 +443,24 @@ body {
   gap: 0.35rem;
 }
 
+/* 用户消息：靠右并限制最大宽度 */
 .msg.user {
   align-self: flex-end;
   align-items: flex-end;
+  max-width: 20%;
 }
 
+/* 助手消息可稍宽 */
 .msg.assistant {
   align-self: flex-start;
+  max-width: 80%;
 }
 
+/* ===== 聊天气泡 ===== */
 .msg-bubble {
-  border-radius: 14px;
-  padding: 0.6rem 0.85rem;
-  border: 1px solid rgba(148, 163, 184, 0.45);
+  border-radius: 18px;                     /* 更明显的圆角 */
+  padding: 0.85rem 1.05rem;                /* 更充足的内边距，让内容不贴边 */
+  border: 1px solid rgba(148, 163, 184, 0.4);
   line-height: 1.55;
   white-space: pre-wrap;
   word-break: break-word;
@@ -466,49 +470,65 @@ body {
   backdrop-filter: blur(12px);
   -webkit-backdrop-filter: blur(12px);
   box-shadow: 0 4px 20px rgba(15, 23, 42, 0.06);
+  overflow: hidden;                        /* 裁剪图片/代码块等内容的直角，保持圆角边界 */
 }
 
 .msg.user .msg-bubble {
   background: var(--bubble-user);
+  text-align: right;
+}
+
+.msg.user .msg-bubble img {
+  margin-left: auto;
+  margin-right: 0;
 }
 
 .msg.assistant .msg-bubble {
   background: var(--bubble-assistant);
 }
 
+/* 图片在气泡内留出呼吸空间 */
 .msg-bubble img {
   display: block;
   width: auto;
   height: auto;
-  max-width: min(360px, 100%);
-  border-radius: 10px;
-  border: 1px solid rgba(148, 163, 184, 0.35);
-  margin-top: 0.25rem;
+  max-width: min(340px, 100%);               /* 缩小占比，不撑满气泡 */
+  border-radius: 12px;                      /* 图片自身圆角，配合气泡圆角 */
+  border: 1px solid rgba(148, 163, 184, 0.3);
+  margin: 0.4rem 0;                         /* 上下对称间距 */
 }
 
 .msg-bubble > img:first-child {
-  margin-top: 0;
+  margin-top: 0.15rem;                      /* 图片在顶部时也保留一点间距 */
 }
 
- .msg-bubble audio {
+.msg-bubble > img:last-child {
+  margin-bottom: 0.15rem;
+}
+
+.msg-bubble audio {
   max-width: 100%;
   display: block;
   margin-top: 0.25rem;
 }
 
+/* ===== 纯音频消息 ===== */
 .msg--audio-only {
-  max-width: min(var(--audio-bubble-max), 82%);
+  max-width: min(var(--audio-bubble-max), 72%);
 }
 
+/* 纯音频消息的气泡：透明无边框，保留音频 pill 尾部空间 */
 .msg--audio-only .msg-bubble {
-  padding: 0.2rem 0;
+  padding: 0.4rem 0.35rem;                  /* 给 pill 两侧留出呼吸空间 */
   border: none;
   background: transparent;
   box-shadow: none;
   backdrop-filter: none;
   -webkit-backdrop-filter: none;
+  overflow: visible;                        /* 允许音频 pill 尾部小三角超出气泡 */
 }
 
+/* ===== 音频胶囊 ===== */
 .audio-pill-wrap {
   display: flex;
   align-items: center;
@@ -764,11 +784,12 @@ body {
   color: #115e59;
 }
 
+/* ===== 富文本消息 ===== */
 .msg-bubble.msg-bubble--rich {
   white-space: normal;
 }
 
- .msg-prose {
+.msg-prose {
   margin: 0.4rem 0;
   font-size: 0.95rem;
 }
@@ -777,7 +798,7 @@ body {
   line-height: 1.65;
 }
 
- .markdown-body p,
+.markdown-body p,
 .markdown-body ul,
 .markdown-body ol,
 .markdown-body blockquote,
@@ -826,7 +847,8 @@ body {
   margin-bottom: 0;
 }
 
- .msg-typing .msg-typing-bubble {
+/* ===== 打字动画 ===== */
+.msg-typing .msg-typing-bubble {
   min-width: 5.5rem;
   padding: 0.65rem 1rem;
 }
@@ -860,33 +882,7 @@ body {
   animation-delay: 0.3s;
 }
 
-@keyframes audioWave {
-  0%,
-  100% {
-    transform: scaleY(0.65);
-    opacity: 0.72;
-  }
-
-  50% {
-    transform: scaleY(1.15);
-    opacity: 1;
-  }
-}
-
-@keyframes typingBounce {
-  0%,
-  60%,
-  100% {
-    opacity: 0.25;
-    transform: translateY(0);
-  }
-
-  30% {
-    opacity: 1;
-    transform: translateY(-3px);
-  }
-}
-
+/* ===== 代码块 ===== */
 .code-block-wrap {
   margin: 0.55rem 0;
   border-radius: 10px;
@@ -949,6 +945,7 @@ body {
   white-space: pre;
 }
 
+/* ===== 输入区域 ===== */
 .composer {
   flex-shrink: 0;
   margin-top: 0.65rem;
@@ -1072,6 +1069,17 @@ body {
   filter: hue-rotate(315deg) saturate(1.2);
 }
 
+.mic-btn.is-auto-input {
+  border-color: rgba(148, 163, 184, 0.35);
+  background: rgba(203, 213, 224, 0.65);
+  box-shadow: none;
+}
+
+.mic-btn.is-auto-input img {
+  filter: grayscale(100%);
+  opacity: 0.65;
+}
+
 .mm-input textarea::placeholder {
   color: #64748b;
 }
@@ -1088,6 +1096,27 @@ body {
   align-items: center;
 }
 
+.composer-auto-input {
+  display: flex;
+  align-items: center;
+  margin-left: auto;
+  gap: 0.5rem;
+  padding: 0.35rem 0;
+}
+
+.auto-input-label {
+  font-size: 0.85rem;
+  color: var(--muted);
+  cursor: pointer;
+}
+
+.auto-input-check {
+  width: 1.2rem;
+  height: 1.2rem;
+  cursor: pointer;
+  accent-color: var(--accent);
+}
+
 .composer-send {
   display: flex;
   flex-direction: column;
@@ -1127,7 +1156,7 @@ body {
   cursor: not-allowed;
 }
 
- .btn-secondary {
+.btn-secondary {
   background: rgba(255, 255, 255, 0.35);
 }
 
@@ -1145,7 +1174,7 @@ body {
   background: rgba(255, 255, 255, 0.2);
 }
 
- .file-btn {
+.file-btn {
   display: inline-flex;
   align-items: center;
   gap: 0.25rem;
@@ -1167,7 +1196,33 @@ body {
   color: var(--text);
 }
 
-/* —— 窄屏 / 手机：侧栏改为抽屉，主区全宽，避免 272px 挤爆布局 —— */
+/* ===== 动画 ===== */
+@keyframes audioWave {
+  0%,
+  100% {
+    transform: scaleY(0.65);
+    opacity: 0.72;
+  }
+  50% {
+    transform: scaleY(1.15);
+    opacity: 1;
+  }
+}
+
+@keyframes typingBounce {
+  0%,
+  60%,
+  100% {
+    opacity: 0.25;
+    transform: translateY(0);
+  }
+  30% {
+    opacity: 1;
+    transform: translateY(-3px);
+  }
+}
+
+/* ===== 窄屏 / 手机 ===== */
 @media (max-width: 768px) {
   .main-header {
     align-items: center;
@@ -1177,6 +1232,11 @@ body {
     display: inline-flex;
   }
 
+  /* 用户消息在移动端适度放宽，避免 20% 过窄 */
+  .msg.user {
+    max-width: 45%;
+  }
+
   .app-root.is-mobile .sidebar {
     position: fixed;
     z-index: 50;
@@ -1266,4 +1326,4 @@ body {
   .chat-messages {
     -webkit-overflow-scrolling: touch;
   }
-}
+}

src/agent/web/app.js

@@ -8,6 +8,7 @@
   const imageInput = $("imageInput");
   const audioInput = $("audioInput");
   const micBtn = $("micBtn");
+  const autoInputCheck = $("autoInputCheck");
   const attachmentPreview = $("attachmentPreview");
   const sendBtn = $("sendBtn");
   const clearBtn = $("clearBtn");
@@ -47,7 +48,21 @@
   let recorderStream = null;
   let recordedChunks = [];
   let recordingMimeType = "";
-  /** 最近一次 renderMessages 的 JSON 可序列化快照，用于发送前拼接带 blob: 的乐观用户消息 */
+  // 自动语音输入相关变量
+  let autoInputEnabled = false;
+  let audioContext = null;
+  let analyser = null;
+  let dataArray = null;
+  let mediaStreamSource = null;
+  let autoInputStream = null;
+  let autoRecordingTimeout = null;
+  let silenceTimeout = null;
+  // 声音阈值配置（已优化）
+  const START_THRESHOLD = 0.18;  // 高于此开始录音（较高阈值）
+  const STOP_THRESHOLD  = 0.06;  // 低于此停止录音（较低阈值，接近静音）
+  const AUTO_START_DELAY = 100;
+  const AUTO_STOP_DELAY = 800;   // 缩短停止延迟
+  /** 最近一次 renderMessages 的 JSON 可序列化快照 */
   let lastRenderedMessages = [];
   let activeAudioController = null;
 
@@ -113,7 +128,6 @@
       .replace(/"/g, """);
   }
 
-  /** 与后端气泡结构一致：纯文字为 string，否则为 parts 数组（顺序：图片 → 文字 → 音频） */
   function buildOptimisticUserContent(text, imageBlobUrls, audioPart) {
     const parts = [];
     (imageBlobUrls || []).forEach((url) => {
@@ -194,12 +208,179 @@
     recorderStream = null;
   }
 
+  function stopAudioContext() {
+    if (audioContext) {
+      try {
+        audioContext.close();
+      } catch {
+        /* ignore */
+      }
+      audioContext = null;
+      analyser = null;
+      dataArray = null;
+      mediaStreamSource = null;
+    }
+  }
+
+  // 改用 RMS 计算音量，更符合物理响度
+  function calculateVolume() {
+    if (!analyser || !dataArray) return 0;
+    const bufferLength = analyser.fftSize;
+    const timeData = new Float32Array(bufferLength);
+    analyser.getFloatTimeDomainData(timeData);
+    let sumSquares = 0;
+    for (let i = 0; i < bufferLength; i++) {
+      sumSquares += timeData[i] * timeData[i];
+    }
+    return Math.sqrt(sumSquares / bufferLength);
+  }
+
+  function handleVolumeDetection() {
+    if (!autoInputEnabled || !analyser) return;
+
+    const volume = calculateVolume();
+    
+    handleVolumeDetection.frameCount = (handleVolumeDetection.frameCount || 0) + 1;
+    if (handleVolumeDetection.frameCount % 10 === 0) {
+      console.log("音量检测:", volume.toFixed(3), "| 状态:", 
+        mediaRecorder?.state || "idle", "| 自动输入:", autoInputEnabled);
+    }
+    
+    if (volume > START_THRESHOLD) {
+      console.log("检测到高分贝，音量:", volume.toFixed(3), "> 阈值:", START_THRESHOLD);
+      if (silenceTimeout) {
+        clearTimeout(silenceTimeout);
+        silenceTimeout = null;
+      }
+      if (!mediaRecorder && !autoRecordingTimeout) {
+        console.log("设置延迟启动录音，延迟:", AUTO_START_DELAY, "ms");
+        autoRecordingTimeout = setTimeout(() => {
+          if (autoInputEnabled && !mediaRecorder) {
+            startMicRecording().catch((e) => {
+              console.error("自动开始录音失败:", e);
+              autoRecordingTimeout = null;
+            });
+          }
+          autoRecordingTimeout = null;
+        }, AUTO_START_DELAY);
+      }
+    } else if (volume < STOP_THRESHOLD && mediaRecorder && mediaRecorder.state === "recording") {
+      console.log("检测到静音，音量:", volume.toFixed(3), "< 阈值:", STOP_THRESHOLD);
+      if (autoRecordingTimeout) {
+        clearTimeout(autoRecordingTimeout);
+        autoRecordingTimeout = null;
+      }
+      if (!silenceTimeout) {
+        console.log("设置延迟停止录音，延迟:", AUTO_STOP_DELAY, "ms");
+        silenceTimeout = setTimeout(() => {
+          if (mediaRecorder && mediaRecorder.state === "recording") {
+            stopMicRecording().catch((e) => {
+              console.error("自动停止录音失败:", e);
+            });
+          }
+          silenceTimeout = null;
+        }, AUTO_STOP_DELAY);
+      }
+    }
+
+    if (autoInputEnabled) {
+      requestAnimationFrame(handleVolumeDetection);
+    }
+  }
+
+  async function startAutoInput() {
+    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+      throw new Error("当前浏览器不支持麦克风");
+    }
+
+    audioContext = new (window.AudioContext || window.webkitAudioContext)();
+    
+    if (audioContext.state === 'suspended') {
+      await audioContext.resume();
+    }
+    
+    analyser = audioContext.createAnalyser();
+    analyser.fftSize = 512;          // 平衡速度与精度
+    dataArray = new Float32Array(analyser.fftSize); // 时域数据
+    mediaStreamSource = null;       // 稍后连接
+
+    autoInputStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    mediaStreamSource = audioContext.createMediaStreamSource(autoInputStream);
+    mediaStreamSource.connect(analyser);
+
+    console.log("自动语音输入已开启，音频上下文状态:", audioContext.state);
+    console.log("音量检测阈值 - 开始:", START_THRESHOLD, "停止:", STOP_THRESHOLD);
+
+    handleVolumeDetection();
+  }
+
+  function stopAutoInput() {
+    if (autoRecordingTimeout) {
+      clearTimeout(autoRecordingTimeout);
+      autoRecordingTimeout = null;
+    }
+    if (silenceTimeout) {
+      clearTimeout(silenceTimeout);
+      silenceTimeout = null;
+    }
+    if (autoInputStream) {
+      autoInputStream.getTracks().forEach((track) => {
+        try {
+          track.stop();
+        } catch {
+          /* ignore */
+        }
+      });
+      autoInputStream = null;
+    }
+    stopAudioContext();
+  }
+
+  function setAutoInputUi(enabled) {
+    if (!micBtn) return;
+    micBtn.classList.toggle("is-auto-input", enabled);
+    micBtn.disabled = enabled;
+    micBtn.title = enabled ? "自动语音输入中（不可用）" : "使用麦克风录音";
+    micBtn.setAttribute("aria-label", enabled ? "自动语音输入中（不可用）" : "使用麦克风录音");
+  }
+
+  function toggleAutoInput(enabled) {
+    autoInputEnabled = enabled;
+    
+    if (enabled) {
+      setAutoInputUi(true);
+      startAutoInput().catch((e) => {
+        console.error("开启自动语音输入失败:", e);
+        autoInputEnabled = false;
+        setAutoInputUi(false);
+        if (autoInputCheck) autoInputCheck.checked = false;
+        alert(e.message || "开启自动语音输入失败");
+      });
+    } else {
+      setAutoInputUi(false);
+      stopAutoInput();
+      if (mediaRecorder && mediaRecorder.state === "recording") {
+        stopMicRecording().catch(() => {});
+      }
+    }
+  }
+
   function setRecordingUi(recording) {
     if (!micBtn) return;
     micBtn.classList.toggle("is-recording", recording);
     micBtn.setAttribute("aria-pressed", recording ? "true" : "false");
-    micBtn.title = recording ? "停止录音" : "使用麦克风录音";
-    micBtn.setAttribute("aria-label", recording ? "停止录音" : "使用麦克风录音");
+    
+    if (recording) {
+      micBtn.title = "停止录音";
+      micBtn.setAttribute("aria-label", "停止录音");
+      micBtn.classList.remove("is-auto-input");
+    } else {
+      micBtn.title = autoInputEnabled ? "自动语音输入中（不可用）" : "使用麦克风录音";
+      micBtn.setAttribute("aria-label", autoInputEnabled ? "自动语音输入中（不可用）" : "使用麦克风录音");
+      if (autoInputEnabled) {
+        micBtn.classList.add("is-auto-input");
+      }
+    }
   }
 
   function inferRecordedExtension(mimeType) {
@@ -234,7 +415,11 @@
           recordedChunks = [];
           recordingMimeType = "";
           mediaRecorder = null;
-          stopRecorderStream();
+          if (!autoInputEnabled) {
+            stopRecorderStream();
+          } else {
+            recorderStream = null;
+          }
           setRecordingUi(false);
         }
       }, { once: true });
@@ -242,7 +427,11 @@
         recordedChunks = [];
         recordingMimeType = "";
         mediaRecorder = null;
-        stopRecorderStream();
+        if (!autoInputEnabled) {
+          stopRecorderStream();
+        } else {
+          recorderStream = null;
+        }
         setRecordingUi(false);
         reject(event.error || new Error("录音失败"));
       }, { once: true });
@@ -273,7 +462,12 @@
       }
     }) || "";
 
-    recorderStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    if (autoInputEnabled && autoInputStream) {
+      recorderStream = autoInputStream;
+    } else {
+      recorderStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    }
+    
     recordedChunks = [];
     recordingMimeType = supportedType;
     mediaRecorder = supportedType
@@ -433,7 +627,6 @@
     return wrap;
   }
 
-  /** 解析 ```lang … ``` 与正文，生成片段（用于助手气泡）。 */
   function parseAssistantContent(full) {
     return renderMarkdownToFragment(full);
   }
@@ -445,7 +638,6 @@
     bubble.appendChild(renderMarkdownToFragment(t));
   }
 
-  /** 整包到达后：先伪流式显示纯文本，再替换为代码块等富结构。 */
   async function pseudoStreamAssistant(bubble, full) {
     const t = full == null ? "" : String(full);
     bubble.classList.add("msg-bubble--rich", "markdown-body");
@@ -489,7 +681,6 @@
     });
   }
 
-  /** 流式渲染新生成的 assistant 消息（允许自动播放音频） */
   async function streamAssistantMessage(msg) {
     const wrap = document.createElement("div");
     wrap.className = "msg assistant" + (isAudioOnlyAssistantMessage(msg) ? " msg--audio-only" : "");
@@ -508,7 +699,6 @@
       await pseudoStreamAssistant(bubble, textParts.map((part) => part.text || "").join("\n"));
     }
 
-    // 媒体部分：显式开启 allowAutoplay
     const mediaOptions = {
       autoplay: true,
       compact: textParts.length === 0,
@@ -562,7 +752,7 @@
       compact = false,
       messageMeta = null,
       role = "assistant",
-      allowAutoplay = false,  // 默认不自动播放，只对新生成消息显式开启
+      allowAutoplay = false,
     } = options;
 
     const wrap = document.createElement("div");
@@ -680,7 +870,6 @@
     wrap.appendChild(button);
     updateProgressUi();
 
-    // 只有 allowAutoplay 为 true 时才自动播放
     const shouldAutoplay = allowAutoplay !== false && Boolean(
       autoplay || part.autoplay || (messageMeta && messageMeta.autoplay)
     );
@@ -748,11 +937,6 @@
     return div;
   }
 
-  /** 
-   * 渲染单条消息
-   * @param {object} msg - 消息对象
-   * @param {boolean} allowAutoplay - 是否允许音频自动播放（仅新生成消息为 true）
-   */
   function renderMessage(msg, allowAutoplay = false) {
     const wrap = document.createElement("div");
     wrap.className = "msg " + (msg.role === "user" ? "user" : "assistant") + (isAudioOnlyAssistantMessage(msg) ? " msg--audio-only" : "");
@@ -796,11 +980,6 @@
     return wrap;
   }
 
-  /** 
-   * 渲染消息列表
-   * @param {Array} messages - 消息数组
-   * @param {boolean} allowAutoplay - 是否允许音频自动播放（默认 false）
-   */
   function renderMessages(messages, allowAutoplay = false) {
     chatMessages.innerHTML = "";
     const list = messages || [];
@@ -843,7 +1022,7 @@
     sessionId = data.sessionId;
     sessions = data.sessions || [];
     renderSessions();
-    renderMessages(data.messages);  // allowAutoplay 默认为 false
+    renderMessages(data.messages);
   }
 
   async function selectSession(id) {
@@ -852,7 +1031,7 @@
     sessionId = data.sessionId;
     sessions = data.sessions || [];
     renderSessions();
-    renderMessages(data.messages);  // 切换会话不自动播放
+    renderMessages(data.messages);
     closeDrawerIfMobile();
   }
 
@@ -866,7 +1045,7 @@
     audioInput.value = "";
     renderSessions();
     renderAttachmentChips();
-    renderMessages(data.messages);  // 新建会话不自动播放
+    renderMessages(data.messages);
     closeDrawerIfMobile();
   }
 
@@ -885,7 +1064,7 @@
     audioInput.value = "";
     renderAttachmentChips();
     renderSessions();
-    renderMessages(data.messages || []);  // 删除后不自动播放
+    renderMessages(data.messages || []);
     closeDrawerIfMobile();
   }
 
@@ -894,7 +1073,7 @@
     const fd = new FormData();
     fd.append("session_id", sessionId);
     const data = await api("/api/sessions/clear", { method: "POST", body: fd });
-    renderMessages(data.messages);  // 清空后不自动播放
+    renderMessages(data.messages);
   }
 
   async function sendChat() {
@@ -925,7 +1104,7 @@
       }
       const optimisticContent = buildOptimisticUserContent(trimmed, imageBlobUrls, audioPart);
       if (optimisticContent != null) {
-        renderMessages([...priorMessages, { role: "user", content: optimisticContent }]);  // 乐观渲染不自动播放
+        renderMessages([...priorMessages, { role: "user", content: optimisticContent }]);
       }
       textInput.value = "";
       pendingImages = [];
@@ -955,7 +1134,7 @@
         renderAttachmentChips();
       }
       renderSessions();
-      renderMessages(startData.messages || []);  // start 返回不自动播放
+      renderMessages(startData.messages || []);
       revokeBlobUrlStack(blobUrls);
       typingEl = appendThinkingIndicator();
 
@@ -974,11 +1153,11 @@
       const last = messages[messages.length - 1];
       if (last && last.role === "assistant") {
         const head = messages.slice(0, -1);
-        renderMessages(head);  // 历史消息不自动播放
+        renderMessages(head);
         revokeBlobUrlStack(blobUrls);
-        await streamAssistantMessage(last);  // 只有新消息流式渲染时自动播放
+        await streamAssistantMessage(last);
       } else {
-        renderMessages(messages);  // 无 assistant 消息时不自动播放
+        renderMessages(messages);
         revokeBlobUrlStack(blobUrls);
       }
     } catch (e) {
@@ -988,7 +1167,7 @@
         pendingImages = savedImages.slice();
         pendingAudio = savedAudio;
         renderAttachmentChips();
-        renderMessages(priorMessages);  // 回滚不自动播放
+        renderMessages(priorMessages);
       }
       if (typingEl && typingEl.parentNode) typingEl.remove();
       alert(e.message || String(e));
@@ -998,12 +1177,12 @@
     }
   }
 
-
   imageInput.addEventListener("change", () => {
     const files = Array.from(imageInput.files || []);
     pendingImages = pendingImages.concat(files);
     imageInput.value = "";
     renderAttachmentChips();
+    textInput.focus();
   });
 
   audioInput.addEventListener("change", () => {
@@ -1011,6 +1190,7 @@
     pendingAudio = f || null;
     audioInput.value = "";
     renderAttachmentChips();
+    textInput.focus();
   });
 
   window.addEventListener("beforeunload", () => {
@@ -1033,6 +1213,7 @@
 
   sendBtn.addEventListener("click", sendChat);
   micBtn?.addEventListener("click", () => toggleMicRecording().catch((e) => alert(e.message || String(e))));
+  autoInputCheck?.addEventListener("change", (ev) => toggleAutoInput(ev.target.checked));
   clearBtn.addEventListener("click", () => clearChat().catch((e) => alert(e.message)));
   newSessionBtn.addEventListener("click", () => newSession().catch((e) => alert(e.message)));
   deleteSessionBtn?.addEventListener("click", () => deleteSession().catch((e) => alert(e.message)));
@@ -1087,14 +1268,13 @@
       }
     });
   }
-  loadBgPrefs();
 
-  /** 背景图原始像素（预加载成功后写入） */
   let bgNaturalW = 0;
   let bgNaturalH = 0;
-  /** @type {ReturnType<typeof setTimeout> | null} */
   let bgFitTimer = null;
 
+  loadBgPrefs();
+
   function backgroundImageHref() {
     try {
       return new URL("image/background.jpg", window.location.href).href;
@@ -1109,7 +1289,6 @@
     return Number.isFinite(n) ? n : 0;
   }
 
-  /** 按「铺满视口」缩放（等价 cover，无黑边；景深略放大以减轻模糊露边） */
   function applyBackgroundFit() {
     if (!bgImageLayer || !bgNaturalW || !bgNaturalH) return;
     const vw = window.innerWidth;
@@ -1131,7 +1310,6 @@
     }, 100);
   }
 
-  /** 与 CSS 中 url("background.jpg") 一致；预加载得到尺寸后再写 background-size */
   function wireBackgroundImage() {
     if (!bgImageLayer) return;
     const href = backgroundImageHref();

src/agent/web/index.html

@@ -115,6 +115,13 @@
                   <option value="audio">纯语音</option>
                   <option value="both">both</option>
                 </select>
+                <!-- 自动语音输入开关 -->
+                <div class="composer-auto-input">
+                  <label class="auto-input-label" for="autoInputCheck">
+                    自动语音输入
+                  </label>
+                  <input type="checkbox" id="autoInputCheck" class="auto-input-check" />
+                </div>
               </div>
             </div>
             <div class="composer-send">

src/agent/web/ui_session.py

@@ -15,7 +15,6 @@ from agent.utils.agent_utils import (
     CHAT_HISTORY_DB_FS,
     MULTI_INPUTS_ROOT,
     assistant_text,
-    extract_result_from_invoke,
 )
 from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS
 from agent.chatbot_agent import get_history
@@ -98,72 +97,34 @@ def _load_ui_messages_from_db(session_id: str) -> list:
         if m.type == "human":
             rows.append({"role": "user", "content": _lc_content_for_ui_messages(m.content)})
         elif m.type in ("ai", "assistant"):
-            text = assistant_text(m.content) or extract_result_from_invoke(m)
+            text = assistant_text(m.content)
             rows.append({"role": "assistant", "content": text or "（空回复）"})
     return rows
 
 
 def _build_user_chat_content(query) -> str | list | None:
+    """根据上传文件扩展名直接生成多模态 content（与 LLM 输入格式一致）"""
     parts: list = []
     for filepath in query.get("files", []) or []:
-        if filepath:
+        if not filepath:
+            continue
+        ext = os.path.splitext(filepath)[1].lower()
+        if ext in IMAGE_FILE_EXTENSIONS:
+            parts.append({"type": "image_url", "image_url": {"url": filepath}})
+        elif ext in _AUDIO_EXTENSIONS:
+            parts.append({"type": "audio", "audio_path": filepath})
+        else:
             parts.append({"path": filepath})
     text = (query.get("text") or "").strip()
     if text:
         parts.append({"type": "text", "text": text})
     if not parts:
         return None
-    if len(parts) == 1 and isinstance(parts[0], dict) and parts[0].get("type") == "text":
+    if len(parts) == 1 and parts[0].get("type") == "text":
         return parts[0]["text"]
     return parts
 
 
-def collect_media_paths_and_query_text(user_msg: dict) -> tuple[list[str], list[str], str]:
-    raw = user_msg.get("content")
-    text_segments: list[str] = []
-    image_paths: list[str] = []
-    audio_paths: list[str] = []
-
-    if isinstance(raw, str):
-        t = raw.strip()
-        if t:
-            text_segments.append(t)
-        return image_paths, audio_paths, "\n".join(text_segments).strip()
-
-    if not isinstance(raw, list):
-        return image_paths, audio_paths, ""
-
-    for piece in raw:
-        if isinstance(piece, str):
-            t = piece.strip()
-            if t:
-                text_segments.append(t)
-            continue
-        if not isinstance(piece, dict):
-            continue
-        if piece.get("type") == "text":
-            text_content = (piece.get("text") or "").strip()
-            if text_content:
-                text_segments.append(text_content)
-            continue
-        file_path = piece.get("path") or (piece.get("file") or {}).get("path")
-        if piece.get("type") == "image_url":
-            file_path = file_path or (piece.get("image_url") or {}).get("url")
-        if piece.get("type") == "audio":
-            file_path = file_path or piece.get("audio_path")
-        if not file_path:
-            continue
-        ext = os.path.splitext(file_path)[1].lower()
-        if ext in IMAGE_FILE_EXTENSIONS:
-            image_paths.append(file_path)
-        elif ext in _AUDIO_EXTENSIONS:
-            audio_paths.append(file_path)
-        else:
-            text_segments.append(f"[用户上传了附件文件: {os.path.basename(file_path)}]")
-
-    return image_paths, audio_paths, "\n".join(text_segments).strip()
-
-
 def rewrite_user_message_files_to_session_dir(user_msg: dict, session_id: str) -> None:
     dest_dir = session_dir(session_id)
     os.makedirs(dest_dir, exist_ok=True)
@@ -304,7 +265,7 @@ def _serialize_assistant_entry(assistant_msg: dict | None) -> dict:
     if isinstance(ac, str):
         atext = ac
     else:
-        atext = assistant_text(ac) or extract_result_from_invoke(assistant_msg) or "（空回复）"
+        atext = assistant_text(ac) or "（空回复）"
     audio_relpath = None
     audio = meta.get("audio")
     if isinstance(audio, dict) and audio.get("path"):
@@ -458,4 +419,4 @@ def clear_ui_and_manifest(session_id: str):
         os.makedirs(os.path.dirname(mp), exist_ok=True)
         with open(mp, "w", encoding="utf-8") as f:
             json.dump({"version": 1, "turns": []}, f, ensure_ascii=False, indent=2)
-    return [], {"text": "", "files": []}
+    return [], {"text": "", "files": []}