diff --git a/.env b/.env index 3db4a82..96a0cc4 100644 --- a/.env +++ b/.env @@ -14,14 +14,14 @@ TTS_BASE_URL = "http://192.168.137.96:8902" RAG_BASE_URL = "http://192.168.137.96:8903" #6000pro： -LLM_API_KEY='semi-ai.tech' -LLM_BASE_URL="http://semi-ai.tech:40056/v1" -LLM_MODEL = "Intel/Qwen3.6-35B-A3B-int4-mixed-AutoRound" +# LLM_API_KEY='semi-ai.tech' +# LLM_BASE_URL="http://semi-ai.tech:40056/v1" +# LLM_MODEL = "Intel/Qwen3.6-35B-A3B-int4-mixed-AutoRound" #A100： -# LLM_API_KEY = 'sk-kGhWAQTqLk9ECsmIDb758a456bD34e92B9771dF2E36432Fb' -# LLM_BASE_URL = "https://app.semi-ai.tech:60004/v1" -# LLM_MODEL = "Qwen3.5-122B-A10B" +LLM_API_KEY = 'sk-kGhWAQTqLk9ECsmIDb758a456bD34e92B9771dF2E36432Fb' +LLM_BASE_URL = "https://app.semi-ai.tech:60004/v1" +LLM_MODEL = "Qwen3.5-122B-A10B" # LLM_TIMEOUT = 300 # LLM_REQUESTS_PER_SECOND = 10.0 # LLM_MAX_BUCKET_SIZE = 10 diff --git a/requirements.txt b/requirements.txt index cd2918c..d6f1aad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # Core LangChain & LangGraph -langchain>=0.1.0 -langchain-core>=0.1.0 -langchain-community>=0.1.0 -langchain-openai>=0.2.0 -langgraph>=1.0.0 -openai>=1.0.0 +langchain>=1.2.0 +langchain-core>=1.2.0 +langchain-community>=0.4.1 +langchain-openai>=1.1.12 +langgraph>=1.1.6 +openai>=2.25.0 # Web framework & server fastapi>=0.115.0 diff --git a/src/agent/__pycache__/chatbot_agent.cpython-312.pyc b/src/agent/__pycache__/chatbot_agent.cpython-312.pyc index 75ed37c..90def3a 100644 Binary files a/src/agent/__pycache__/chatbot_agent.cpython-312.pyc and b/src/agent/__pycache__/chatbot_agent.cpython-312.pyc differ diff --git a/src/agent/__pycache__/llm.cpython-312.pyc b/src/agent/__pycache__/llm.cpython-312.pyc index b0d46de..a2a3a65 100644 Binary files a/src/agent/__pycache__/llm.cpython-312.pyc and b/src/agent/__pycache__/llm.cpython-312.pyc differ diff --git a/src/agent/__pycache__/llm.cpython-313.pyc b/src/agent/__pycache__/llm.cpython-313.pyc new file mode 100644 index 0000000..182d0ee Binary files /dev/null and b/src/agent/__pycache__/llm.cpython-313.pyc differ diff --git a/src/agent/chatbot_agent.py b/src/agent/chatbot_agent.py index a24329e..db96ed0 100644 --- a/src/agent/chatbot_agent.py +++ b/src/agent/chatbot_agent.py @@ -1,15 +1,18 @@ -# agent/chatbot_agent.py -from typing import Any +from typing import Any, Literal, Sequence, Annotated, TypedDict +import json -from langchain_core.messages import HumanMessage, AIMessage -from langchain.agents import create_agent +from langgraph.graph import StateGraph, START, END +from langgraph.prebuilt import ToolNode +from langgraph.graph.message import add_messages +from langchain_core.messages import BaseMessage, HumanMessage, AIMessage, ToolMessage -from agent.llm import Qwen36_35B_A3B_FP8_LLM +from agent.utils.image_utils import image_bytes_to_openai_image_url_part +from agent.llm import llm from agent.tools import get_tools from agent.utils.agent_utils import ( assistant_text, get_history, - summarizer, # 使用新的上下文感知构建器 + summarizer, get_session_meta, update_session_meta, ) @@ -20,26 +23,152 @@ from web.ui_session import ( ) from agent.memory.system_prompt import MAIN_SYSTEM_PROMPT -# ---------- 1. 工具 ---------- +# ---------- 1. 工具与模型 ---------- tools = get_tools() +model_with_tools = llm.bind_tools(tools) -# ---------- 2. 创建 ReAct agent（LangGraph）---------- -agent_graph = create_agent( - model=Qwen36_35B_A3B_FP8_LLM, - tools=tools, - name="main" -) +# ---------- 2. 定义状态 ---------- +class AgentState(TypedDict): + messages: Annotated[Sequence[BaseMessage], add_messages] -# ---------- 3. 辅助函数：保存消息到数据库 ---------- +# ---------- 3. 节点定义 ---------- +def call_model(state: AgentState) -> dict: + """调用 LLM 生成回复或工具调用""" + response = model_with_tools.invoke(state["messages"]) + + # ----- 打印 token 使用情况 ----- + usage = response.usage_metadata if hasattr(response, 'usage_metadata') else {} + if usage: + input_tokens = usage.get('input_tokens', '?') + output_tokens = usage.get('output_tokens', '?') + total_tokens = usage.get('total_tokens', '?') + print(f"[LLM] Token使用: input={input_tokens}, output={output_tokens}, total={total_tokens}") + + # ----- 打印工具调用信息 ----- + if hasattr(response, 'tool_calls') and response.tool_calls: + for tc in response.tool_calls: + tool_name = tc.get('name', 'unknown') + args = tc.get('args', {}) + # LangChain 中 tool_call 的 args 可能是 JSON 字符串，尝试解析 + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + pass + print(f"[LLM] 工具调用: {tool_name}, 参数: {args}") + + return {"messages": [response]} + +# 工具执行节点（LangGraph 内置，自动解包 content_and_artifact 到 ToolMessage.artifact） +tool_node = ToolNode(tools) + +def process_tool_artifact(state: AgentState) -> dict: + last_msg = state["messages"][-1] + + # 日志打印 + if isinstance(last_msg, ToolMessage): + content_preview = str(last_msg.content) + if len(content_preview) > 200: + content_preview = content_preview[:200] + '...' + print(f"[Tool] 工具返回 (id={last_msg.tool_call_id}): {content_preview}") + + # 只处理 ToolMessage 且有 artifact 的情况 + if not isinstance(last_msg, ToolMessage) or not last_msg.artifact: + return {} + + artifact = last_msg.artifact + + # ---------- 统一提取所有 image_bytes ---------- + img_bytes_list: list[bytes] = [] + + if isinstance(artifact, dict) and "image_bytes" in artifact: + # 单张图片：artifact = {"image_bytes": b"..."} + img_bytes_list.append(artifact["image_bytes"]) + + elif isinstance(artifact, list): + for item in artifact: + if isinstance(item, dict) and "image_bytes" in item: + img_bytes_list.append(item["image_bytes"]) + elif isinstance(item, bytes): + img_bytes_list.append(item) + + # 兼容 artifact 直接就是 bytes 的情况 + elif isinstance(artifact, bytes): + img_bytes_list.append(artifact) + + if not img_bytes_list: + return {} + + # ---------- 将图片转换为多模态 content ---------- + content_parts = [] + failed_count = 0 + + for img_bytes in img_bytes_list: + data_url = image_bytes_to_openai_image_url_part(img_bytes) + if data_url is None: + failed_count += 1 + continue + content_parts.append(data_url) + + # 所有图片都处理失败时的降级方案 + if not content_parts: + human_msg = HumanMessage( + content="工具返回了截图，但图片处理失败，请向用户说明。", + additional_kwargs={"synthetic": True} + ) + return {"messages": [human_msg]} + + # 追加引导文本（放在最后，也可以放在最前，取决于你的需求） + content_parts.append({ + "type": "text", + "text": "上面是工具返回的截图。" + }) + + # 如果有部分图片失败，可以在 text 中提示（可选） + if failed_count > 0: + content_parts[-1]["text"] += f"（其中 {failed_count} 张图片处理失败，请忽略）" + + human_msg = HumanMessage( + content=content_parts, + additional_kwargs={"synthetic": True} + ) + return {"messages": [human_msg]} + +# ---------- 4. 条件路由 ---------- +def should_continue(state: AgentState) -> Literal["tools", END]: + last_msg = state["messages"][-1] + if hasattr(last_msg, "tool_calls") and last_msg.tool_calls: + return "tools" + return END + +# ---------- 5. 构建 Graph ---------- +workflow = StateGraph(AgentState) + +workflow.add_node("agent", call_model) +workflow.add_node("tools", tool_node) +workflow.add_node("process_tool_artifact", process_tool_artifact) + +workflow.add_edge(START, "agent") +workflow.add_conditional_edges("agent", should_continue, { + "tools": "tools", + END: END, +}) +workflow.add_edge("tools", "process_tool_artifact") +workflow.add_edge("process_tool_artifact", "agent") + +agent_graph = workflow.compile() + +# ---------- 6. 辅助函数：保存消息（过滤合成消息） ---------- def save_messages_to_history(session_id: str, user_msg: HumanMessage, assistant_msg: AIMessage) -> None: - """将用户消息和助手消息追加到 SQLite 历史记录中""" if not session_id: return history = get_history(session_id) - history.add_message(user_msg) + # 只保存非合成消息 + if not user_msg.additional_kwargs.get("synthetic"): + history.add_message(user_msg) history.add_message(assistant_msg) -# ---------- 4. 核心执行函数 ---------- +# ---------- 7. 核心执行函数 ---------- def execute_agent( chat_history: list[dict[str, Any]], session_id: str = "test", @@ -54,20 +183,18 @@ def execute_agent( if session_id: rewrite_user_message_files_to_session_dir(chat_history[-1], session_id) - # 直接使用 chat_history 中已经正确格式化的 content（由 _build_user_chat_content 生成） user_content = chat_history[-1]["content"] - # 动态上下文感知构建消息 + # 上下文感知构建 messages, context_exceeded = summarizer( session_id=session_id, - agent_name="main", # 固定 agent 名称，可扩展 + agent_name="main", system_prompt=MAIN_SYSTEM_PROMPT, user_content=user_content, - llm=Qwen36_35B_A3B_FP8_LLM, + llm=llm, ) if context_exceeded: - # 上下文已满，返回提示信息 chat_history.append({ "role": "assistant", "content": "上下文已满，请新建会话继续对话。", @@ -76,7 +203,6 @@ def execute_agent( sync_manifest_from_chat_history(session_id, chat_history, "") return chat_history - # 调用 graph 的配置 graph_config = { "configurable": {"session_id": session_id}, "metadata": { @@ -86,15 +212,15 @@ def execute_agent( } try: - # 调用 LangGraph agent + # 调用自定义 Graph result = agent_graph.invoke({"messages": messages}, config=graph_config) - # 从最终消息中提取文本回复 + # 提取最后一条 AI 消息 all_messages = result["messages"] assistant_msg = all_messages[-1] if all_messages else AIMessage(content="") text_content = assistant_text(assistant_msg.content) - # 提取 token 使用量并更新到 meta + # token 统计更新 usage = assistant_msg.usage_metadata if hasattr(assistant_msg, 'usage_metadata') else {} total_tokens = usage.get('total_tokens', 0) if isinstance(usage, dict) else 0 if total_tokens > 0: @@ -107,9 +233,9 @@ def execute_agent( last_total_tokens=total_tokens, ) - # 处理 TTS 音频数据 + # TTS 处理（保持原逻辑） output_type_from_config = graph_config.get("metadata", {}).get("output_type", "text") - audio_data = Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs or {} + audio_data = llm.last_audio_kwargs or {} if audio_data.get("output_type") != output_type_from_config: audio_data = {} @@ -119,7 +245,6 @@ def execute_agent( "audio": audio_data.get("audio"), } - # 根据输出类型构建前端 content if output_type_from_config == "audio" and assistant_meta.get("audio"): assistant_content = [] elif output_type_from_config == "both" and assistant_meta.get("audio"): @@ -127,22 +252,20 @@ def execute_agent( else: assistant_content = text_content if text_content else "（空回复）" - # 追加到返回的 chat_history chat_history.append({ "role": "assistant", "content": assistant_content, "meta": assistant_meta, }) - # 持久化历史消息 + # 持久化历史消息（注意过滤掉合成消息） save_messages_to_history( session_id, HumanMessage(content=user_content), AIMessage(content=text_content), ) - # 清空音频缓存 - Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs = None + llm.last_audio_kwargs = None except Exception as e: import traceback @@ -151,7 +274,6 @@ def execute_agent( err_text = "抱歉，我遇到了一些问题，请稍后再试。" chat_history.append({"role": "assistant", "content": err_text}) - # 同步文件清单 if session_id: sync_manifest_from_chat_history(session_id, chat_history, "") diff --git a/src/agent/history/data_base/chat_history.db b/src/agent/history/data_base/chat_history.db index fbf8476..d96b9db 100644 Binary files a/src/agent/history/data_base/chat_history.db and b/src/agent/history/data_base/chat_history.db differ diff --git a/src/agent/history/multi_inputs/20260519_144316/ui_messages.json b/src/agent/history/multi_inputs/20260519_144316/ui_messages.json deleted file mode 100644 index 9721091..0000000 --- a/src/agent/history/multi_inputs/20260519_144316/ui_messages.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "version": 2, - "turns": [ - { - "user": { - "content": [ - { - "type": "text", - "text": "你好" - } - ] - }, - "assistant": { - "text": "你好！很高兴能帮到你。\n\n如果你有图片、文件需要解读，或者有任何问题想要咨询，随时可以发给我或告诉我。请问有什么我可以帮你的吗？", - "output_type": "text", - "autoplay": false, - "audio_duration_seconds": null, - "audio_relpath": null - } - } - ] -} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260527_130428/ui_messages.json b/src/agent/history/multi_inputs/20260527_130428/ui_messages.json new file mode 100644 index 0000000..a2d4907 --- /dev/null +++ b/src/agent/history/multi_inputs/20260527_130428/ui_messages.json @@ -0,0 +1,22 @@ +{ + "version": 2, + "turns": [ + { + "user": { + "content": [ + { + "type": "text", + "text": "\"C:\\Users\\86166\\Desktop\\rag_test\"，帮我把这个文件夹下的文档全部归入rag个人知识库" + } + ] + }, + "assistant": { + "text": "已成功将 `C:\\Users\\86166\\Desktop\\rag_test` 文件夹下的所有文档（共4个文件）添加到您的RAG个人知识库中，系统共提取并添加了11条知识块。\n\n包含的文件如下：\n- 曾嘉翔-用户画像.docx\n- 朱振宁-用户画像.docx\n- 赵永顺-今日行程安排.docx\n- 赵永顺-用户画像.docx\n\n如果您需要查询这些内容或进行其他操作，请随时告诉我！", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260527_130748/ui_messages.json b/src/agent/history/multi_inputs/20260527_130748/ui_messages.json new file mode 100644 index 0000000..550b169 --- /dev/null +++ b/src/agent/history/multi_inputs/20260527_130748/ui_messages.json @@ -0,0 +1,56 @@ +{ + "version": 2, + "turns": [ + { + "user": { + "content": [ + { + "type": "text", + "text": "赵永顺今天上午吃了什么" + } + ] + }, + "assistant": { + "text": "我无法知道赵永顺今天上午吃了什么，因为我不了解他的个人生活细节，也没有相关的实时信息。如果您是他本人或认识他，可以直接询问他本人。", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null + } + }, + { + "user": { + "content": [ + { + "type": "text", + "text": "你可以去rag库查一查" + } + ] + }, + "assistant": { + "text": "根据工具返回的文本信息，赵永顺今天上午10点5分醒来后，吃了**两个酱肉包**。\n\n之后他进行了rag开发测试，并计划下午剪辑照片、召开讨论和分享会。", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null + } + }, + { + "user": { + "content": [ + { + "type": "text", + "text": "那他的系统架构是什么样的呢" + } + ] + }, + "assistant": { + "text": "根据这张截图，赵永顺的系统架构是一个名为 **`chatbot_agent`** 的编排系统，其核心是一个多模态智能体，主要包含以下几个部分：\n\n1. **用户层 (User Layer)**\n * **Gradio 桌面 UI**: 提供桌面应用程序的交互界面。\n * **FastAPI Web 静态前端**: 提供基于Web的交互界面。\n * 这两个前端都作为用户入口，与核心的 `chatbot_agent` 进行交互。\n\n2. **chatbot_agent 编排 (Orchestration)**\n 这是系统的核心处理逻辑，具体流程如下：\n * **用户输入**: 接收来自 `MultimodalTextbox` 或 `Chatbot` 的消息。\n * **消息记录**: 通过 `add_message` 将用户消息写入对话记录。\n * **代理执行 (`execute_agent`)**: 这是核心逻辑处理单元。\n * **图片处理**: 首先判断输入中是否包含图片。\n * **有图片**: 调用 `describe_images` (基于 Qwen-VL 多模态模型) 生成图片描述 (`image_description`)。\n * **无图片**: 直接使用纯文本输入。\n * **输入组合**: 将图片描述（如果有）和用户查询 (`query`) 组合成纯文本 (`chain_input_from_image_and_query`)。\n * **文件处理**: 用户上传的文件会被复制到 `history/multi_inputs/(session_id)/` 目录下。\n * **上下文组合**: 将图片路径、图片描述和用户查询组合成最终的输入 (`chain_input`)。\n * **历史会话管理**:\n * **读取历史**: 从 SQLite 数据库中读取会话历史 (`summarize_history`)。\n * **摘要与截断**: 如果消息数量超过8条，系统会将旧消息生成一个对话摘要，并只保留最近的8条消息 (`chat_history`)。\n * **构建完整上下文**: 将系统提示词 (包含助手人设和摘要占位符) 和对话历史 (`chat_history`) 组合在一起 (`assign`)。\n * **绑定历史**: 使用 `RunnableWithMessageHistory` 将构建好的上下文与历史会话绑定，并准备将新的交互写回数据库。\n\n3. **模型与外部 (Model & External)**\n * 系统通过一个 **ChatOpenAI 兼容客户端** 与外部的 **Qwen35 网关** (以 OpenAI API 风格) 进行交互，调用大语言模型。\n\n4. **持久化 (Persistence)**\n * **SQLite**: 用于存储 `chat_history.db` 和 `message_store`。\n * **ui_messages.json**: 存储会话清单。\n * **multi_inputs 附件目录**: 存储用户上传的文件。\n\n**总结来说**，这是一个结构清晰的智能体应用架构，它通过LangChain进行编排，能够处理多模态输入（文本和图片），智能地管理长对话历史（通过摘要和截断），并持久化存储所有会话和附件信息，最终通过API调用外部的大模型来生成回复。", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null + } + } + ] +} \ No newline at end of file diff --git a/src/agent/llm.py b/src/agent/llm.py index 800cac0..3ea43b8 100644 --- a/src/agent/llm.py +++ b/src/agent/llm.py @@ -9,20 +9,16 @@ from typing import Any, List, Dict, Optional from pydantic import Field import re, json from datetime import datetime as dt - import requests + from langchain_core.messages import HumanMessage, AIMessage, BaseMessage, SystemMessage from langchain_core.outputs import ChatResult, ChatGeneration from langchain_openai import ChatOpenAI from agent.utils.image_utils import image_bytes_to_openai_image_url_part from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, TTS_BASE_URL, ASR_BASE_URL -from agent.utils.agent_utils import assistant_text -from agent.utils.image_utils import ( - image_path_to_openai_image_url_part, - image_bytes_to_openai_image_url_part, - IMAGE_FILE_EXTENSIONS, -) +from agent.utils.agent_utils import assistant_text, Documents_process +from agent.utils.image_utils import image_path_to_openai_image_url_part, image_bytes_to_openai_image_url_part, IMAGE_FILE_EXTENSIONS from agent.memory.system_prompt import VISION_SYSTEM_PROMPT class ChatQwen(ChatOpenAI): @@ -186,10 +182,26 @@ class ChatQwen(ChatOpenAI): file_path = part.get("path") or (part.get("file") or {}).get("path") if isinstance(file_path, str) and os.path.isfile(file_path): ext = os.path.splitext(file_path)[1].lower() + normalized_path = file_path.replace("\\", "/") if ext in IMAGE_FILE_EXTENSIONS: new_content.append(self._process_image_content({"type": "image_url", "image_url": {"url": file_path}})) else: - new_content.append({"type": "text", "text": f"[用户上传了附件文件: {os.path.basename(file_path)}]"}) + try: + file_text, file_images = Documents_process.process(file_path) + new_content.append({"type": "text", "text": f"用户上传了一个名为\"{normalized_path}\"的文件。"}) + if file_text.strip(): + new_content.append({ + "type": "text", + "text": f"以下是文件 {normalized_path} 的内容：\n{file_text.strip()}" + }) + for image_bytes in file_images or []: + image_part = image_bytes_to_openai_image_url_part(image_bytes) + if image_part is not None: + new_content.append(image_part) + if not file_text.strip() and not file_images: + new_content.append({"type": "text", "text": f"[用户上传了附件文件: {normalized_path}]"}) + except Exception as e: + new_content.append({"type": "text", "text": f"[附件处理失败: {normalized_path}: {e}]"}) else: text = assistant_text(part) if text: @@ -365,8 +377,7 @@ class ChatQwen(ChatOpenAI): result = self.invoke(messages) return result.content - -Qwen36_35B_A3B_FP8_LLM = ChatQwen() +llm = ChatQwen() if __name__ == "__main__": import sys @@ -382,7 +393,7 @@ if __name__ == "__main__": print("=" * 50) print("测试1: 纯图片识别（无文本）") print("=" * 50) - result1 = Qwen36_35B_A3B_FP8_LLM.invoke([ + result1 = llm.invoke([ HumanMessage(content=[{"type": "image_url", "image_url": {"url": TEST_IMAGE_PATH}}]) ]) print(f"返回类型: {type(result1)}") @@ -393,7 +404,7 @@ if __name__ == "__main__": print("\n" + "=" * 50) print("测试2: 图文混合输入") print("=" * 50) - result2 = Qwen36_35B_A3B_FP8_LLM.invoke([ + result2 = llm.invoke([ HumanMessage(content=[ {"type": "image_url", "image_url": {"url": TEST_IMAGE_PATH}}, {"type": "text", "text": "这张图片里有什么？请详细描述"} diff --git a/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/data_level0.bin b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/data_level0.bin new file mode 100644 index 0000000..b3f8c53 Binary files /dev/null and b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/data_level0.bin differ diff --git a/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/header.bin b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/header.bin new file mode 100644 index 0000000..b4a33c1 Binary files /dev/null and b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/header.bin differ diff --git a/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/length.bin b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/length.bin new file mode 100644 index 0000000..83f0686 Binary files /dev/null and b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/length.bin differ diff --git a/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/link_lists.bin b/src/agent/memory/chroma_rag_db/93b83aa9-b3da-41a4-b34c-90fcd21bda19/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/src/agent/memory/chroma_rag_db/chroma.sqlite3 b/src/agent/memory/chroma_rag_db/chroma.sqlite3 new file mode 100644 index 0000000..57f28d4 Binary files /dev/null and b/src/agent/memory/chroma_rag_db/chroma.sqlite3 differ diff --git a/src/agent/tools/__init__.py b/src/agent/tools/__init__.py index e36bd43..d4f507b 100644 --- a/src/agent/tools/__init__.py +++ b/src/agent/tools/__init__.py @@ -24,9 +24,10 @@ from .tool_rag import RAGTool _all_tools = [ Websearchclass(), DateQueryClass(), - ScreenCaptureDescribeClass(), + ScreenCapture(), BrowserControlClass(), RAGTool(), + DirectoryListing(), ] def get_tools(): diff --git a/src/agent/tools/__pycache__/__init__.cpython-312.pyc b/src/agent/tools/__pycache__/__init__.cpython-312.pyc index e386175..a6bb083 100644 Binary files a/src/agent/tools/__pycache__/__init__.cpython-312.pyc and b/src/agent/tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/tool.cpython-312.pyc b/src/agent/tools/__pycache__/tool.cpython-312.pyc index 8d69275..a3b68b7 100644 Binary files a/src/agent/tools/__pycache__/tool.cpython-312.pyc and b/src/agent/tools/__pycache__/tool.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/tool_rag.cpython-312.pyc b/src/agent/tools/__pycache__/tool_rag.cpython-312.pyc index 23609a6..a120e87 100644 Binary files a/src/agent/tools/__pycache__/tool_rag.cpython-312.pyc and b/src/agent/tools/__pycache__/tool_rag.cpython-312.pyc differ diff --git a/src/agent/tools/tool.py b/src/agent/tools/tool.py index ff2e9f5..2eb4114 100644 --- a/src/agent/tools/tool.py +++ b/src/agent/tools/tool.py @@ -1,16 +1,19 @@ from __future__ import annotations +import os import io -from typing import Optional, Literal +from typing import Optional, Literal, Tuple import sys import time import webbrowser import pyautogui +import asyncio +from pathlib import Path from langchain.tools import BaseTool from pydantic import BaseModel, Field, create_model -from agent.llm import Qwen36_35B_A3B_FP8_LLM +from agent.llm import llm from utils.env_utils import BROWSER_MAP @@ -56,8 +59,8 @@ class DateQueryClass(BaseTool): return self._run() -class ScreenCaptureDescribeClass(BaseTool): - """截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。""" +class ScreenCapture(BaseTool): + """截取当前屏幕，生成文字描述并同时提供原始截图供模型直接查看。""" args_schema: type[BaseModel] = create_model( "ScreenCaptureInput", @@ -69,44 +72,42 @@ class ScreenCaptureDescribeClass(BaseTool): ), ), ) - name: str = "screen_capture_describe" + name: str = "screen_capture" description: str = ( - "截取当前主屏幕画面，并用视觉模型转为文字描述。" + "截取当前主屏幕画面。除了返回文字描述外，也会提供原始截图给模型进行视觉分析。" "注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。" ) + response_format: Literal["content", "content_and_artifact"] = "content_and_artifact" - def _run(self, focus_query: Optional[str] = None) -> str: - # 1. 确保 pyautogui 可用 - try: - import pyautogui - except ImportError as e: - return ( - f"未安装 pyautogui，无法截屏: {e}。" - "请在项目环境中执行: pip install pyautogui" - ) - - # 2. 截屏并转为 PNG 字节流 + def _run(self, focus_query: Optional[str] = None) -> Tuple[str, dict]: + # 1. 截屏 try: pil_img = pyautogui.screenshot() buf = io.BytesIO() pil_img.save(buf, format="PNG") raw_png = buf.getvalue() except Exception as e: - return f"截屏失败: {e}" + return f"截屏失败: {e}", {} - # 3. 构建描述指令 query = (focus_query or "").strip() - - # 4. 调用与对话图片上传完全相同的视觉模型进行描述 - try: - # Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义 - text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query) - except Exception as e: - return f"屏幕图像视觉解析失败: {e}" - if not (text or "").strip(): - return "（已截屏，但模型未返回有效描述。）" - return f"【当前屏幕视觉描述】\n{text.strip()}" + if query: # focus_query 存在且非空 + # 生成视觉描述，返回文字内容，不返回图片字节 + try: + description = llm.describe_image_bytes(raw_png, prompt=query) + except Exception as e: + description = f"屏幕图像视觉解析失败: {e}" + if not description.strip(): + description = "（未返回有效描述）" + content_text = f"【当前屏幕视觉描述】\n{description.strip()}" + return content_text, {} + else: + # 不生成描述，直接返回图片字节 + artifact = {"image_bytes": raw_png} + return "", artifact + + async def _arun(self, focus_query: Optional[str] = None) -> Tuple[str, dict]: + return await asyncio.to_thread(self._run, focus_query) class BrowserControlClass(BaseTool): """统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL""" @@ -133,21 +134,19 @@ class BrowserControlClass(BaseTool): "- action='open', url='current': 刷新当前标签页\n" "- action='open', url='https://...': 在新标签页打开指定 URL\n" "- action='close', url=None: 关闭整个浏览器窗口\n" - "- action='close', url='current': 关闭当前标签页\n" - "- action='close', url='https://...': 关闭当前活动标签页" + "- action='close', url='任意非空字符(例如current)': 关闭当前标签页\n" + "操作完成后会返回当前屏幕截图，帮助判断操作是否成功。" "仅用于进行浏览器操作时调用，其余任何情况严禁调用。" ) + response_format: Literal["content", "content_and_artifact"] = "content_and_artifact" - # ---------- 辅助方法 ---------- + # ---------- 辅助方法（与原版相同，无改动） ---------- def _get_browser_info(self, browser_name: Optional[str]): - """获取浏览器控制器和窗口标题关键字""" if browser_name is None: return webbrowser.get(), None - name_lower = browser_name.lower().strip() if name_lower not in BROWSER_MAP: raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}") - reg_name, title_keyword = BROWSER_MAP[name_lower] try: controller = webbrowser.get(reg_name) @@ -156,7 +155,6 @@ class BrowserControlClass(BaseTool): return controller, title_keyword def _activate_window(self, title_keyword: str) -> bool: - """激活浏览器窗口（仅 Windows）""" if sys.platform != "win32": return False try: @@ -173,24 +171,18 @@ class BrowserControlClass(BaseTool): return False def _send_keys(self, *keys): - """发送快捷键，根据平台调整""" pyautogui.hotkey(*keys) - # ---------- 核心操作 ---------- - def _open_action(self, url: Optional[str], browser: Optional[str]): - """执行打开类操作""" + # ---------- 核心操作（返回操作结果描述） ---------- + def _open_action(self, url: Optional[str], browser: Optional[str]) -> str: controller, title_keyword = self._get_browser_info(browser) - if url is None: - # 仅打开浏览器首页（激活或启动） controller.open("about:blank") if title_keyword: self._activate_window(title_keyword) browser_info = f" (使用 {browser})" if browser else "" return f"已打开浏览器首页{browser_info}" - elif url.lower() == "current": - # 刷新当前页 if title_keyword: if not self._activate_window(title_keyword): return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新" @@ -200,20 +192,15 @@ class BrowserControlClass(BaseTool): else: self._send_keys("f5") return "已刷新当前标签页" - else: - # 打开指定 URL controller.open(url) if title_keyword: self._activate_window(title_keyword) return f"已尝试打开 URL: {url}" - def _close_action(self, url: Optional[str], browser: Optional[str]): - """执行关闭类操作""" + def _close_action(self, url: Optional[str], browser: Optional[str]) -> str: _, title_keyword = self._get_browser_info(browser) - if url is None: - # 关闭整个浏览器 if title_keyword: if not self._activate_window(title_keyword): return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭" @@ -223,9 +210,7 @@ class BrowserControlClass(BaseTool): else: self._send_keys("alt", "f4") return "已尝试关闭浏览器窗口" - - elif url.lower() == "current": - # 关闭当前标签页 + else: if title_keyword: if not self._activate_window(title_keyword): return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页" @@ -236,16 +221,82 @@ class BrowserControlClass(BaseTool): self._send_keys("ctrl", "w") return "已尝试关闭当前标签页" - def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str: + # ---------- 同步主逻辑，返回 (描述, artifact) ---------- + def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> Tuple[str, dict]: try: if action == "open": - return self._open_action(url, browser) + msg = self._open_action(url, browser) elif action == "close": - return self._close_action(url, browser) + msg = self._close_action(url, browser) else: - return f"不支持的操作类型: {action}" + msg = f"不支持的操作类型: {action}" except Exception as e: - return f"执行浏览器操作时出错: {e}" + msg = f"执行浏览器操作时出错: {e}" + + # 无论操作是否成功，都尝试截屏 + try: + pil_img = pyautogui.screenshot() + buf = io.BytesIO() + pil_img.save(buf, format="PNG") + raw_png = buf.getvalue() + return msg, {"image_bytes": raw_png} + except Exception as e: + return f"{msg}\n(截屏失败: {e})", {} + + # ---------- 异步入口 ---------- + async def _arun(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> Tuple[str, dict]: + return await asyncio.to_thread(self._run, action, url, browser) + +class DirectoryListing(BaseTool): + """列出指定目录下的文件和文件夹，帮助 Agent 了解本地文件结构。""" + + args_schema: type[BaseModel] = create_model( + "DirectoryListingInput", + path=( + Optional[str], + Field( + default=None, + description="要查看的目录路径，默认为当前工作目录。" + ), + ), + show_hidden=( + bool, + Field( + default=False, + description="是否显示隐藏文件（以点开头的文件/文件夹），默认不显示。" + ), + ), + ) + name: str = "directory_listing" + description: str = ( + "列出指定目录下的所有文件和子目录。可用于探索文件系统、查找文件位置。" + "仅当需要获取目录内容或确认文件结构时调用，其余情况严禁调用。" + ) + + def _run(self, path: Optional[str] = None, show_hidden: bool = False) -> str: + try: + target = Path(path).resolve() if path else Path.cwd() + if not target.exists(): + return f"路径不存在: {target}" + if not target.is_dir(): + return f"路径不是目录: {target}" + + items = [] + with os.scandir(target) as entries: + for entry in entries: + if not show_hidden and entry.name.startswith('.'): + continue + item_type = "DIR" if entry.is_dir() else "FILE" + items.append(f"[{item_type}] {entry.name}") + + if not items: + return f"目录 {target} 为空" + return f"目录 {target} 的内容:\n" + "\n".join(sorted(items)) + except Exception as e: + return f"列出目录时出错: {str(e)}" + + async def _arun(self, path: Optional[str] = None, show_hidden: bool = False) -> str: + return await asyncio.to_thread(self._run, path, show_hidden) if __name__ == "__main__": # 这里可以进行一些简单的测试，例如创建工具实例并调用方法 diff --git a/src/agent/tools/tool_rag.py b/src/agent/tools/tool_rag.py index dce1e2a..a395a31 100644 --- a/src/agent/tools/tool_rag.py +++ b/src/agent/tools/tool_rag.py @@ -1,11 +1,12 @@ # agent/tools/tool_rag.py from __future__ import annotations +import asyncio +import re import hashlib -import json import os from datetime import datetime -from typing import List, Optional, Literal, Union +from typing import List, Optional, Literal, Union, Tuple import chromadb import requests @@ -14,24 +15,113 @@ from pydantic import BaseModel, Field, create_model from langchain_text_splitters import RecursiveCharacterTextSplitter from utils.env_utils import RAG_BASE_URL, RAG_CHUNK_SIZE, RAG_CHUNK_OVERLAP +from utils.agent_utils import Documents_process +from ..llm import llm -# Chroma 持久化路径：agent/memory/chroma_rag_db +# ---------- 前置函数---------- +ENDING_PUNCTUATION = set(".!?。！？…\"'”’）;:；：…") + + +def has_ending_punctuation(text: str) -> bool: + stripped = text.rstrip() + if not stripped: + return True + return stripped[-1] in ENDING_PUNCTUATION + + +def pre_process(text_content: str, list_image_byte: List[bytes]) -> Tuple[str, List[bytes]]: + lines = text_content.split('\n') + new_lines = [] + i = 0 + while i < len(lines): + line = lines[i] + stripped = line.strip() + # 情形1：当前行是 ... + if stripped.startswith('') and stripped.endswith(''): + if i + 1 < len(lines): + next_line = lines[i + 1] + next_stripped = next_line.strip() + # 如果下一行不是 且没有结束标点，合并 + if (next_stripped and + not has_ending_punctuation(next_stripped) and + not next_stripped.startswith('')): + merged = f'{next_stripped}' + new_lines.append(merged) + i += 2 + continue + new_lines.append(line) + i += 1 + # 情形2：当前行是普通文本，没有结束标点 + elif (stripped and + not has_ending_punctuation(stripped) and + not stripped.startswith('')): + if i + 1 < len(lines): + next_line = lines[i + 1] + next_stripped = next_line.strip() + # 如果下一行是 ...，合并 + if next_stripped.startswith('') and next_stripped.endswith(''): + merged = f'{stripped}' + new_lines.append(merged) + i += 2 + continue + new_lines.append(line) + i += 1 + else: + new_lines.append(line) + i += 1 + + merged_text = '\n'.join(new_lines) + + # 替换 内容：插入图片描述 + pattern = re.compile(r'(.*?)', re.DOTALL) + matches = list(pattern.finditer(merged_text)) + result_parts = [] + last_end = 0 + for idx, match in enumerate(matches): + result_parts.append(merged_text[last_end:match.start()]) + inner = match.group(1).strip() + if idx < len(list_image_byte): + try: + description = llm.describe_image_bytes( + list_image_byte[idx], + prompt="""你是一名专业的图像理解助手,现在需要你为即将加入向量库的图片做文字分析以便于用户更迅速、更高效、较准确地查询到此图片。 + 首先请用客观、准确的中文描述用户提供的图片，覆盖主要物体、场景、人物动作、界面元素、图表数据与清晰可读的文字； + 涉及到你认识的具体品牌、时间、地点、人物、影视剧、物品等特定内容时请直接命名说明；看不清的内容请如实说明，不要臆测。 + 其次请为此图片设想出3到5个可能会问及的客观问题,放在末尾处。""" + ) + except Exception: + description = "图片描述生成失败" + else: + description = "" + if inner: + new_inner = f"{inner}\n{description}" if description else inner + else: + new_inner = description + result_parts.append(f"{new_inner}") + last_end = match.end() + result_parts.append(merged_text[last_end:]) + final_text = ''.join(result_parts) + return final_text, list_image_byte + + +# ---------- 存储路径、客户端 ---------- CHROMA_PERSIST_DIR = os.path.join(os.path.dirname(__file__), '..', 'memory', 'chroma_rag_db') +# 统一的图片存储目录 +IMAGE_STORE_DIR = os.path.join(CHROMA_PERSIST_DIR, "images") -# 全局 Chroma 客户端和统一集合 _chroma_client = None _global_collection = None - GLOBAL_COLLECTION_NAME = "rag_global" + def get_chroma_client(): global _chroma_client if _chroma_client is None: _chroma_client = chromadb.PersistentClient(path=CHROMA_PERSIST_DIR) return _chroma_client + def get_global_collection(): - """获取或创建全局知识库 Collection""" global _global_collection if _global_collection is None: client = get_chroma_client() @@ -46,11 +136,8 @@ def get_global_collection(): ) return _global_collection -# ---------- Embedding 接口封装 ---------- + def embed_texts(queries: List[str], documents: List[str] = None) -> tuple: - """ - 调用 /embed，返回 (query_embeddings, doc_embeddings) - """ payload = { "task_description": "Given a web search query, retrieve relevant passages that answer the query", "queries": queries, @@ -64,8 +151,8 @@ def embed_texts(queries: List[str], documents: List[str] = None) -> tuple: except Exception as e: raise RuntimeError(f"Embedding 请求失败: {e}") + def rerank_documents(query: str, documents: List[str]) -> List[float]: - """调用 /rerank，返回排序后的分数列表""" payload = { "task_description": "Given a web search query, retrieve relevant passages that answer the query", "query": query, @@ -78,93 +165,49 @@ def rerank_documents(query: str, documents: List[str]) -> List[float]: except Exception as e: raise RuntimeError(f"Reranker 请求失败: {e}") -# ---------- 文本分块 ---------- -def split_into_chunks(text: str) -> List[str]: - splitter = RecursiveCharacterTextSplitter( - chunk_size=RAG_CHUNK_SIZE, - chunk_overlap=RAG_CHUNK_OVERLAP, - separators=["\n\n", "\n", " ", ""] - ) - return splitter.split_text(text) - -# ---------- 文件解析 ---------- -def parse_file_to_chunks(file_path: str) -> List[dict]: - """ - 解析文件，返回每个 chunk 的字典，包含： - - content: 文本内容 - - source: 详细来源描述 - """ - ext = os.path.splitext(file_path)[1].lower() - chunks = [] - - if ext == ".txt" or ext == ".md": - with open(file_path, "r", encoding="utf-8") as f: - text = f.read() - texts = split_into_chunks(text) - for i, t in enumerate(texts): - chunks.append({ - "content": t, - "source": f"{os.path.basename(file_path)} (chunk {i+1}/{len(texts)})" - }) - - elif ext == ".json": - with open(file_path, "r", encoding="utf-8") as f: - data = json.load(f) - text = json.dumps(data, ensure_ascii=False, indent=2) - texts = split_into_chunks(text) - for i, t in enumerate(texts): - chunks.append({ - "content": t, - "source": f"{os.path.basename(file_path)} (JSON, chunk {i+1}/{len(texts)})" - }) - - elif ext == ".csv": - import pandas as pd - df = pd.read_csv(file_path) - text = df.to_string(index=False) - texts = split_into_chunks(text) - for i, t in enumerate(texts): - chunks.append({ - "content": t, - "source": f"{os.path.basename(file_path)} (CSV, chunk {i+1}/{len(texts)})" - }) - - elif ext == ".docx": - from docx import Document - doc = Document(file_path) - full_text = "\n".join([p.text for p in doc.paragraphs]) - texts = split_into_chunks(full_text) - for i, t in enumerate(texts): - chunks.append({ - "content": t, - "source": f"{os.path.basename(file_path)} (DOCX, chunk {i+1}/{len(texts)})" - }) - - elif ext == ".pdf": - import pdfplumber - with pdfplumber.open(file_path) as pdf: - for page_num, page in enumerate(pdf.pages, start=1): - page_text = page.extract_text() or "" - if page_text.strip(): - page_chunks = split_into_chunks(page_text) - for i, t in enumerate(page_chunks): - chunks.append({ - "content": t, - "source": f"{os.path.basename(file_path)} (Page {page_num}, chunk {i+1})" - }) +def split_into_chunks(input_data) -> List[dict]: + if isinstance(input_data, str): + text = input_data + image_bytes_list = [] + elif isinstance(input_data, tuple) and len(input_data) == 2: + text, image_bytes_list = input_data else: - raise ValueError(f"不支持的文件格式: {ext}") + text = "" + image_bytes_list = [] - return chunks + pattern = re.compile(r'([\s\S]*?|[\s\S]*?
)') + parts = pattern.split(text) + all_chunks = [] + image_index = 0 + for part in parts: + if not part: + continue + if (part.startswith('') and part.endswith('')) or \ + (part.startswith('') and part.endswith('
')): + chunk_info = {"content": part, "is_protected": True} + if part.startswith('') and image_index < len(image_bytes_list): + chunk_info["image_bytes"] = image_bytes_list[image_index] + image_index += 1 + all_chunks.append(chunk_info) + else: + splitter = RecursiveCharacterTextSplitter( + chunk_size=RAG_CHUNK_SIZE, + chunk_overlap=RAG_CHUNK_OVERLAP, + separators=["\n\n", "\n", "。", "！", "？", "；", "，", ".", "!", "?", ";", ",", " ", ""] + ) + split_parts = splitter.split_text(part) + all_chunks.extend([{"content": p, "is_protected": False} for p in split_parts]) + return all_chunks -# ---------- RAG 工具本体 ---------- + +# ---------- RAGTool ---------- class RAGTool(BaseTool): name: str = "rag_tool" description: str = ( - "通用知识库工具，支持增删查操作。\n" + "个人知识库管理工具，支持增删查及列出所有来源操作。\n" "参数说明：\n" - "- action_type: 操作类型，必填。可选：'query'(查询), 'add_document'(添加文档), 'clear'(清除)。\n" + "- action_type: 操作类型，必填。可选：'query'(查询), 'add_document'(添加文档), 'clear'(清除), 'list_sources'(列出所有来源)。\n" "- content: 查询内容(用于query) 或 直接文本内容(用于add_document，与source二选一)。\n" "- source: 文件路径(用于add_document，与content二选一)。\n" "- filter: 过滤条件(用于query或clear)。\n" @@ -172,24 +215,29 @@ class RAGTool(BaseTool): "- 查询：action_type='query', content='用户问题'\n" "- 添加文本：action_type='add_document', content='知识文本'\n" "- 添加文件：action_type='add_document', source='/path/to/file.pdf'\n" - "- 清除：action_type='clear', filter='某个来源'" + "- 清除：action_type='clear', filter='某个来源'\n" + "- 列出所有来源：action_type='list_sources'" ) args_schema: type[BaseModel] = create_model( "RAGInput", - action_type=(Literal["query", "add_document", "clear"], Field(..., description="操作类型")), + action_type=(Literal["query", "add_document", "clear", "list_sources"], Field(..., description="操作类型")), content=(Optional[str], Field(None, description="查询内容或直接添加的文本内容")), - source=(Optional[str], Field(None, description="文件路径，用于添加文档")), + source=(Optional[Union[str, List[str]]], Field(None, description="文件路径（字符串或列表），用于添加文档")), filter=(Optional[Union[str, List[str]]], Field(None, description="过滤条件，用于查询或清除")) ) + # ========== 关键改动：声明返回内容和 artifact ========== + response_format: Literal["content", "content_and_artifact"] = "content_and_artifact" + def _run( self, action_type: str, content: Optional[str] = None, - source: Optional[str] = None, + source: Optional[Union[str, List[str]]] = None, filter: Optional[Union[str, List[str]]] = None - ) -> str: + ) -> Tuple[str, Union[dict, list]]: + """统一返回 (text, artifact) 元组""" try: if action_type == "query": return self._handle_query(content, filter) @@ -197,27 +245,47 @@ class RAGTool(BaseTool): return self._handle_add(content, source) elif action_type == "clear": return self._handle_clear(filter) + elif action_type == "list_sources": + return self._handle_list_sources() else: - return f"未知操作: {action_type}" + return f"未知操作: {action_type}", {} except Exception as e: - return f"RAG 工具执行出错: {str(e)}" + return f"RAG 工具执行出错: {str(e)}", {} - # ---------- 查询 ---------- - def _handle_query(self, query: str, filter: Optional[Union[str, List[str]]]) -> str: + async def _arun(self, **kwargs) -> Tuple[str, Union[dict, list]]: + return await asyncio.to_thread(self._run, **kwargs) + + def _get_batch_chunks(self, coll, batch_id: str) -> dict: + res = coll.get(where={"batch_id": batch_id}) + batch_chunks = {} + if res and res["metadatas"] and res["documents"]: + for meta, doc in zip(res["metadatas"], res["documents"]): + chunk_idx = meta.get("chunk_index") + if chunk_idx is not None: + batch_chunks[chunk_idx] = { + "content": doc, + "is_image": meta.get("is_image", False), + "is_table": meta.get("is_table", False), + "is_protected": meta.get("is_protected", False), + "image_path": meta.get("image_path", "") + } + return batch_chunks + + # ---------- 查询（返回 artifact）---------- + def _handle_query(self, query: str, filter: Optional[Union[str, List[str]]]) -> Tuple[str, list]: if not query: - return "查询内容不能为空。" + return "查询内容不能为空。", [] # 1. 获取查询向量 q_embs, _ = embed_texts(queries=[query]) query_vector = q_embs[0] - # 2. 从全局 Collection 中检索 + # 2. 检索 coll = get_global_collection() where = None if filter: if isinstance(filter, str): where = {"source": {"$contains": filter}} - # 列表暂简化为只支持单个字符串，但保留接口 elif isinstance(filter, list) and len(filter) > 0: where = {"source": {"$contains": filter[0]}} @@ -228,7 +296,7 @@ class RAGTool(BaseTool): ) if not res["ids"] or not res["ids"][0]: - return "未在知识库中找到相关内容。" + return "未在知识库中找到相关内容。", [] ids = res["ids"][0] docs = res["documents"][0] @@ -243,7 +311,7 @@ class RAGTool(BaseTool): unique_results = all_results[:10] - # 3. Reranker 精排 + # 3. Reranker docs_for_rerank = [r["document"] for r in unique_results] scores = rerank_documents(query, docs_for_rerank) @@ -254,93 +322,240 @@ class RAGTool(BaseTool): )[:3] if not sorted_items: - return "Reranker 未返回有效结果。" + return "Reranker 未返回有效结果。", [] outputs = [] + all_image_bytes = [] # 收集图片字节 + for item, score in sorted_items: meta = item["metadata"] - source_info = meta.get("source", "未知来源") - outputs.append(f"【来源：{source_info}（相关度 {score:.3f}）】\n{item['document']}") + is_image = meta.get("is_image", False) + is_table = meta.get("is_table", False) + batch_id = meta.get("batch_id") + chunk_index = meta.get("chunk_index") - return "\n\n".join(outputs) + batch_chunks = self._get_batch_chunks(coll, batch_id) if batch_id else {} + + # 拼接上下文（前、当前、后） + context_parts = [] + for idx_offset in [-1, 0, 1]: + current_idx = chunk_index + idx_offset + if current_idx in batch_chunks: + chunk = batch_chunks[current_idx] + context_parts.append(chunk["content"]) + # 如果是图片块，从文件读取字节 + if chunk["is_image"] and chunk.get("image_path"): + img_path = chunk["image_path"] + if os.path.isfile(img_path): + try: + with open(img_path, "rb") as f: + all_image_bytes.append(f.read()) + except Exception: + pass + + context_text = "\n\n".join(context_parts) + + # 格式化文本输出 + if is_image: + outputs.append( + f"【图片数据 - 来源：{meta.get('source', '未知来源')}（相关度 {score:.3f}）】\n[图片数据已获取]") + elif is_table: + outputs.append( + f"【表格数据 - 来源：{meta.get('source', '未知来源')}（相关度 {score:.3f}）】\n{context_text}") + else: + outputs.append( + f"【文本数据 - 来源：{meta.get('source', '未知来源')}（相关度 {score:.3f}）】\n{context_text}") + + final_text = "\n\n".join(outputs) + + # 去重图片字节，并按每个图片单独打包成一个 dict，放入 artifact 列表 + unique_image_bytes = list(dict.fromkeys(all_image_bytes)) # 保持顺序去重 + artifact = [{"image_bytes": img_bytes} for img_bytes in unique_image_bytes] + + return final_text, artifact # ---------- 添加文档 ---------- - def _handle_add(self, content: Optional[str], source: Optional[str]) -> str: + def _handle_add(self, content: Optional[str], source: Optional[Union[str, List[str]]]) -> Tuple[str, dict]: if not content and not source: - return "必须提供 content 或 source。" + return "必须提供 content 或 source。", {} if content and source: - return "content 和 source 只能提供一个。" + return "content 和 source 只能提供一个。", {} - # 1. 准备 chunks 列表 - chunks_meta = [] # 元素为 (text, source_desc) + # 统一 batch_id，整个添加操作共享 timestamp = datetime.now().isoformat() + batch_id = hashlib.md5(timestamp.encode()).hexdigest() + # 收集所有文档的 chunk 信息 + all_chunks_meta = [] # (text, src_desc, is_protected, image_path) + file_errors = [] + + # 处理直接文本输入 if content: - texts = split_into_chunks(content) - chunks_meta = [(t, "直接输入") for t in texts] + processed_text, processed_images = pre_process(content, []) + chunk_infos = split_into_chunks((processed_text, processed_images)) + base_dir = IMAGE_STORE_DIR + base_name = f"direct_{batch_id[:8]}" + source_desc_prefix = "直接输入" + + image_idx = 0 + for idx, chunk_info in enumerate(chunk_infos): + text = chunk_info["content"] + img_bytes = chunk_info.get("image_bytes") + is_protected = chunk_info.get("is_protected", False) + src_desc = f"{source_desc_prefix} (chunk {idx + 1}/{len(chunk_infos)})" + + image_path = "" + if text.startswith('') and text.endswith('') and img_bytes is not None: + os.makedirs(base_dir, exist_ok=True) + image_path = os.path.join(base_dir, f"{base_name}_image{image_idx}.jpg") + try: + with open(image_path, "wb") as f: + f.write(img_bytes) + except Exception as e: + return f"图片保存失败: {e}", {} + image_idx += 1 + + all_chunks_meta.append((text, src_desc, is_protected, image_path)) + + # 处理文件输入（字符串或列表） elif source: - if not os.path.isfile(source): - return f"文件不存在: {source}" - chunks_meta = [(c["content"], c["source"]) for c in parse_file_to_chunks(source)] + # 统一为列表处理 + source_files = [source] if isinstance(source, str) else source + for file_path in source_files: + if not os.path.isfile(file_path): + file_errors.append(f"文件不存在: {file_path}") + continue - if not chunks_meta: - return "文档内容为空，未添加任何条目。" + try: + raw_text, image_list = Documents_process.process(file_path) + except Exception as e: + file_errors.append(f"处理文件失败 {file_path}: {e}") + continue - # 2. 批量获取文档向量 - doc_texts = [cm[0] for cm in chunks_meta] + processed_text, processed_images = pre_process(raw_text, image_list) + chunk_infos = split_into_chunks((processed_text, processed_images)) + if not chunk_infos: + file_errors.append(f"文件内容为空: {file_path}") + continue + + base_dir = os.path.dirname(file_path) + base_name = os.path.splitext(os.path.basename(file_path))[0] + source_desc_prefix = os.path.basename(file_path) + + image_idx = 0 + for idx, chunk_info in enumerate(chunk_infos): + text = chunk_info["content"] + img_bytes = chunk_info.get("image_bytes") + is_protected = chunk_info.get("is_protected", False) + src_desc = f"{source_desc_prefix} (chunk {idx + 1}/{len(chunk_infos)})" + + image_path = "" + if text.startswith('') and text.endswith('') and img_bytes is not None: + os.makedirs(IMAGE_STORE_DIR, exist_ok=True) + image_path = os.path.join(IMAGE_STORE_DIR, f"{base_name}_image{image_idx}.jpg") + try: + with open(image_path, "wb") as f: + f.write(img_bytes) + except Exception as e: + file_errors.append(f"图片保存失败 ({file_path}): {e}") + # 即便图片保存失败，文本块仍可入库，继续执行 + image_idx += 1 + + all_chunks_meta.append((text, src_desc, is_protected, image_path)) + + if not all_chunks_meta: + error_msg = "没有可用内容添加到知识库。" + if file_errors: + error_msg += " 错误详情: " + "; ".join(file_errors) + return error_msg, {} + + # 统一写入 Chroma：分配连续 chunk_index + total_chunks = len(all_chunks_meta) + doc_texts = [m[0] for m in all_chunks_meta] _, doc_embs = embed_texts(queries=["dummy"], documents=doc_texts) - # 3. 写入全局 Collection coll = get_global_collection() - ids = [] - embeddings = [] - metadatas = [] - documents = [] + ids, embeddings, metadatas, documents = [], [], [], [] - for idx, (text, src_desc) in enumerate(chunks_meta): - uid = hashlib.md5(f"{timestamp}_{text[:80]}".encode()).hexdigest() + for idx, (text, src_desc, is_protected, image_path) in enumerate(all_chunks_meta): + uid = hashlib.md5(f"{timestamp}_{text[:80]}_{idx}".encode()).hexdigest() ids.append(uid) embeddings.append(doc_embs[idx]) - metadatas.append({ + + metadata = { "source": src_desc, "timestamp": timestamp, - }) + "batch_id": batch_id, + "chunk_index": idx, # 全局连续索引 + "total_chunks": total_chunks, + "is_image": text.startswith('') and text.endswith(''), + "is_table": text.startswith('') and text.endswith('
'), + "is_protected": is_protected + } + if image_path: + metadata["image_path"] = image_path + + metadatas.append(metadata) documents.append(text) - coll.add( - ids=ids, - embeddings=embeddings, - metadatas=metadatas, - documents=documents - ) + coll.add(ids=ids, embeddings=embeddings, metadatas=metadatas, documents=documents) - return f"成功添加 {len(ids)} 条知识块到全局知识库。" + result_msg = f"成功添加 {total_chunks} 条知识块到全局知识库。" + if file_errors: + result_msg += " 部分文件处理出错: " + "; ".join(file_errors) + + return result_msg, {} # ---------- 清除 ---------- - def _handle_clear(self, filter: Optional[Union[str, List[str]]]) -> str: + def _handle_clear(self, filter: Optional[Union[str, List[str]]]) -> Tuple[str, dict]: if not filter: - return "清除操作必须提供 filter 条件。" + return "清除操作必须提供 filter 条件。", {} coll = get_global_collection() where = None if isinstance(filter, str): where = {"source": {"$contains": filter}} elif isinstance(filter, list) and len(filter) > 0: - # 简化为使用第一个条件 where = {"source": {"$contains": filter[0]}} if not where: - return "无法构建有效的过滤条件。" + return "无法构建有效的过滤条件。", {} try: res = coll.get(where=where) if res["ids"]: + # 清理关联的图片文件 + for meta in res.get("metadatas", []): + img_path = meta.get("image_path", "") + if img_path and os.path.isfile(img_path): + try: + os.remove(img_path) + except Exception: + pass coll.delete(ids=res["ids"]) - return f"已从全局知识库中删除 {len(res['ids'])} 条记录。" + return f"已从全局知识库中删除 {len(res['ids'])} 条记录。", {} else: - return "未找到匹配的记录。" + return "未找到匹配的记录。", {} except Exception as e: - return f"清除时出错: {e}" + return f"清除时出错: {e}", {} - async def _arun(self, **kwargs): - return self._run(**kwargs) \ No newline at end of file + # ---------- 列出所有来源 ---------- + def _handle_list_sources(self) -> Tuple[str, dict]: + """获取知识库中所有不重复的 source 信息""" + coll = get_global_collection() + try: + all_data = coll.get() # 获取全部文档的元数据 + except Exception as e: + return f"获取知识库数据失败: {e}", {} + + if not all_data or not all_data["metadatas"]: + return "知识库中暂无数据。", {} + + sources = set() + for meta in all_data["metadatas"]: + src = meta.get("source", "未知来源") + sources.add(src) + + sorted_sources = sorted(sources) + text = "知识库中的来源列表：\n" + "\n".join(f"- {s}" for s in sorted_sources) + return text, {} \ No newline at end of file diff --git a/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc b/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc index cb78161..74782b5 100644 Binary files a/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc and b/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc differ diff --git a/src/agent/utils/__pycache__/image_utils.cpython-312.pyc b/src/agent/utils/__pycache__/image_utils.cpython-312.pyc index 46cf7dc..00adb64 100644 Binary files a/src/agent/utils/__pycache__/image_utils.cpython-312.pyc and b/src/agent/utils/__pycache__/image_utils.cpython-312.pyc differ diff --git a/src/agent/utils/agent_utils.py b/src/agent/utils/agent_utils.py index e9b4ee1..995fc64 100644 --- a/src/agent/utils/agent_utils.py +++ b/src/agent/utils/agent_utils.py @@ -4,6 +4,8 @@ import os import shutil import sqlite3 from typing import List, Tuple, Any +import pathlib +import mimetypes from langchain_community.chat_message_histories import SQLChatMessageHistory from langchain_core.messages import BaseMessage, SystemMessage, HumanMessage @@ -16,6 +18,17 @@ from agent.utils.env_utils import ( HARD_COMPRESS_RATE, CURRENT_TURNS, ) +from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS + +# 扩展名到内部类型标签的映射 +DOC_EXT_MAP = { + ".txt": "plain_text", ".json": "plain_text", ".md": "plain_text",".cpp": "plain_text", + ".c": "plain_text", ".py": "plain_text", ".m": "plain_text", ".java": "plain_text", + ".docx": "docx", + ".pdf": "pdf", + ".csv": "tabular", + ".xlsx": "tabular", +} # ---------- 目录与数据库路径 ---------- _AGENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -117,12 +130,6 @@ def _count_turns(messages: List[BaseMessage]) -> int: i += 1 return turns -def _last_turn_end_index(messages: List[BaseMessage]) -> int: - for i in range(len(messages) - 1, -1, -1): - if messages[i].type in ("ai", "assistant"): - return i - return -1 - # ---------- 压缩函数 ---------- def _lightweight_compress(llm, messages_to_compress: List[BaseMessage]) -> str: """轻量压缩：保留关键事实、实体、决策，删除冗余""" @@ -281,3 +288,271 @@ def assistant_text(value) -> str: return str(value).strip() return str(value).strip() + +class Documents_process: + """文件分类与内容提取工具集""" + + @staticmethod + def doc_classify(file_path: str) -> str: + """ + 根据扩展名和 MIME 类型返回内部处理类型标签。 + 可能的返回值：'plain_text'、'docx'、'pdf'、'image'、'tabular'。 + """ + ext = pathlib.Path(file_path).suffix.lower() + # 先检查文档映射表 + if ext in DOC_EXT_MAP: + return DOC_EXT_MAP[ext] + + # 再检查是否为图片 + if ext in IMAGE_FILE_EXTENSIONS: + return "image" + + # 最后使用 MIME 类型判断 + mime, _ = mimetypes.guess_type(file_path) + if mime: + if mime.startswith("text/"): + return "plain_text" + if mime.startswith("image/"): + return "image" + + raise ValueError(f"不支持的文件类型: {file_path}") + + # ---------- 各类文件的具体处理方法 ---------- + @staticmethod + def process_plain_text(file_path: str) -> tuple[str, list]: + with open(file_path, "r", encoding="utf-8") as f: + return f.read(), [] + + @staticmethod + def process_docx(file_path: str) -> tuple[str, list]: + """按文档顺序提取文本、图片和表格，并将短标题与图表合并。""" + try: + from docx import Document + from docx.oxml.ns import qn + from docx.opc.constants import RELATIONSHIP_TYPE as RT + except ImportError: + raise ImportError("需要安装 python-docx: pip install python-docx") + + doc = Document(file_path) + content_parts = [] + image_bytes_list = [] + image_counter = 0 + + # ---------- 辅助函数：按文档顺序迭代段落和表格 ---------- + def iter_block_items(parent): + """生成文档中的段落和表格，保持原始顺序""" + from docx.oxml.ns import qn as ns_qn + if hasattr(parent, 'element'): + parent_elm = parent.element.body + else: + parent_elm = parent + + # 建立元素到段落的映射，便于 O(1) 查找 + para_map = {p._element: p for p in doc.paragraphs} + # 表格同样建立映射 + table_map = {t._element: t for t in doc.tables} + + for child in parent_elm.iterchildren(): + if child.tag == ns_qn('w:p'): + para = para_map.get(child) + if para is not None: + yield para + elif child.tag == ns_qn('w:tbl'): + table = table_map.get(child) + if table is not None: + yield table + + def extract_image_from_run(run, para): + """从 run 中提取图片，返回 N 字符串和图片二进制列表""" + nonlocal image_counter + drawings = run._element.findall('.//' + qn('w:drawing')) + imgs = [] + for drawing in drawings: + blips = drawing.findall('.//' + qn('a:blip')) + for blip in blips: + embed = blip.get(qn('r:embed')) + if embed: + image_counter += 1 + related_part = para.part.related_parts[embed] + image_bytes_list.append(related_part.blob) + imgs.append(f"{image_counter}") + return imgs + + pending_title = "" # 缓存可能存在的图表短标题 + + for item in iter_block_items(doc): + if hasattr(item, 'text'): # 段落 + para = item + para_parts = [] + # 提取图片 + for run in para.runs: + para_parts.extend(extract_image_from_run(run, para)) + # 提取文字 + text = para.text.strip() + if text: + para_parts.insert(0, text) + para_str = "".join(para_parts) + + if not para_str: + continue + + # 判断是否是短标题（如“表1 一个表格”、“图1 一张图片”） + if len(text) < 30 and (text.startswith("表") or text.startswith("图") or text.startswith("Table") or text.startswith("Fig")): + # 暂存为候选标题，等待后续表格/图片 + pending_title = para_str + continue + + # 普通段落：先输出可能遗留的标题（没有后续图表匹配的），再输出本段 + if pending_title: + content_parts.append(pending_title + "\n") + pending_title = "" + content_parts.append(para_str + "\n") + + else: # 表格 + table = item + table_str_parts = [f"===== Table ??? =====\n"] # 标题稍后合并 + for row in table.rows: + row_data = [cell.text.strip() for cell in row.cells] + table_str_parts.append("| " + " | ".join(row_data) + " |\n") + table_content = "".join(table_str_parts) + + # 如果有暂存标题，合并到一起 + if pending_title: + # 替换表格占位标题 + final_table = f"{pending_title}\n{table_content[table_content.index('\n')+1:]}
" + content_parts.append(final_table + "\n") + pending_title = "" + else: + # 没有标题，保留原始表格内容 + content_parts.append(f"{table_content}
\n") + + # 文件末尾的遗留标题 + if pending_title: + content_parts.append(pending_title + "\n") + + return "".join(content_parts), image_bytes_list + + @staticmethod + def process_pdf(file_path: str) -> tuple[str, list]: + try: + from pypdf import PdfReader + except ImportError: + raise ImportError("需要安装 pypdf:pip install pypdf") + + reader = PdfReader(file_path) + text_parts = [] + image_bytes_list = [] + image_counter = 0 # 全局图片计数器 + + for page in reader.pages: + page_content = [] + # 提取文字 + page_text = page.extract_text() + if page_text: + page_content.append(page_text) + + # 提取图片，并在文本中追加占位符 + for _ in page.images: + image_counter += 1 + page_content.append(f"{image_counter}") + + # 收集图片二进制数据 + for image_file_object in page.images: + image_bytes_list.append(image_file_object.data) + + # 合并当前页的内容 + if page_content: + text_parts.append("".join(page_content)) + + return "\n".join(text_parts), image_bytes_list + + @staticmethod + def process_tabular(file_path: str) -> tuple[str, list]: + """ + 处理 CSV 和 Excel(.xlsx) 文件。 + 将所有 sheet 的内容转化为 SheetName\n内容
格式的字符串。 + 图片列表固定为空。 + """ + try: + import csv + from openpyxl import load_workbook + except ImportError: + raise ImportError("需要安装 csv和openpyxl:pip install csv openpyxl") + + result_parts = [] + ext = pathlib.Path(file_path).suffix.lower() + + if ext == '.csv': + # CSV 只有一个“sheet” + with open(file_path, 'r', encoding='utf-8-sig', newline='') as f: + reader = csv.reader(f) + rows = list(reader) + table_str = '\n'.join([' | '.join(row) for row in rows]) + result_parts.append(f"Sheet1\n{table_str}
") + + elif ext == '.xlsx': + wb = load_workbook(file_path, read_only=True, data_only=True) + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + rows = [] + for row in ws.iter_rows(values_only=True): + rows.append(' | '.join([str(cell) if cell is not None else '' for cell in row])) + table_str = '\n'.join(rows) + result_parts.append(f"{sheet_name}\n{table_str}
") + wb.close() + else: + raise ValueError(f"Unsupported tabular file extension: {ext}") + + return "\n".join(result_parts), [] + + @staticmethod + def process_image(file_path: str) -> tuple[str, list]: + with open(file_path, "rb") as f: + return "", [f.read()] + + # ---------- 统一处理入口 ---------- + @staticmethod + def process(file_path: str) -> tuple[str, list]: + file_type = Documents_process.doc_classify(file_path) + + if file_type == "plain_text": + return Documents_process.process_plain_text(file_path) + elif file_type == "docx": + return Documents_process.process_docx(file_path) + elif file_type == "pdf": + return Documents_process.process_pdf(file_path) + elif file_type == "image": + return Documents_process.process_image(file_path) + elif file_type == "tabular": + return Documents_process.process_tabular(file_path) + else: + raise ValueError(f"未处理的文件类型: {file_type}") + +if __name__ == "__main__": + docx_path = r"C:\Users\86166\Desktop\Agent_Learning_minor\信息科学与工程学院-赵永顺-《兼爱》——墨翟一生的“白月光”.docx" + pdf_path = r"C:\Users\86166\Desktop\Agent_Learning_minor\信息科学与工程学院-赵永顺-《兼爱》——墨翟一生的“白月光”.pdf" + xlsx_path = r"C:\Users\86166\Desktop\Agent_Learning_minor\电科二班2025德育评价.xlsx" + py_path = r"C:\Users\86166\Desktop\Agent_Learning_minor\speech_server.py" + + output_path = "test_results.txt" + with open(output_path, "w", encoding="utf-8") as f: + test_files = [ + ("py", py_path), + ("docx", docx_path), + ("xlsx", xlsx_path), + ("pdf", pdf_path), + ] + for label, path in test_files: + f.write(f"\n===== 测试 {label} 文件：{path} =====\n") + try: + text, images = Documents_process.process(path) + f.write(f"提取文本长度：{len(text)}\n") + f.write(f"提取图片数量：{len(images)}\n") + f.write("文本预览：\n") + f.write(text) + except Exception as e: + f.write(f"❌ 处理失败: {e}\n") + print(f"测试结果已写入 {output_path}") + + + \ No newline at end of file diff --git a/src/web/__pycache__/server.cpython-312.pyc b/src/web/__pycache__/server.cpython-312.pyc index 6fd3311..9592cae 100644 Binary files a/src/web/__pycache__/server.cpython-312.pyc and b/src/web/__pycache__/server.cpython-312.pyc differ diff --git a/src/web/__pycache__/server.cpython-313.pyc b/src/web/__pycache__/server.cpython-313.pyc new file mode 100644 index 0000000..3575862 Binary files /dev/null and b/src/web/__pycache__/server.cpython-313.pyc differ diff --git a/src/web/__pycache__/ui_session.cpython-312.pyc b/src/web/__pycache__/ui_session.cpython-312.pyc index e70b30c..ae1e597 100644 Binary files a/src/web/__pycache__/ui_session.cpython-312.pyc and b/src/web/__pycache__/ui_session.cpython-312.pyc differ diff --git a/src/web/__pycache__/ui_session.cpython-313.pyc b/src/web/__pycache__/ui_session.cpython-313.pyc new file mode 100644 index 0000000..fed1c4c Binary files /dev/null and b/src/web/__pycache__/ui_session.cpython-313.pyc differ diff --git a/src/web/app.css b/src/web/app.css index c6a04e5..d6250d3 100644 --- a/src/web/app.css +++ b/src/web/app.css @@ -964,6 +964,59 @@ body { min-height: 0; } +.file-btn { + position: relative; +} + +.file-btn--icon-only { + width: 2.1rem; + min-width: 2.1rem; + height: 2.1rem; + padding: 0; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 999px; +} + +.file-btn-icon { + width: 0.9rem; + height: 0.9rem; + display: block; +} + +.document-badge { + display: inline-flex; + align-items: center; + gap: 0.45rem; + max-width: 100%; + margin: 0.3rem 0; + padding: 0.1rem 0; + border: none; + background: transparent; + box-shadow: none; +} + +.document-badge--user { + margin-left: auto; +} + +.document-badge-icon { + width: 0.95rem; + height: 0.95rem; + flex-shrink: 0; + border: none; + margin: 0; +} + +.document-badge-name { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 0.88rem; +} + .chip { font-size: 0.75rem; padding: 0.22rem 0.5rem; diff --git a/src/web/app.js b/src/web/app.js index 060e9cf..a17e553 100644 --- a/src/web/app.js +++ b/src/web/app.js @@ -5,8 +5,7 @@ const chatMessages = $("chatMessages"); const chatPlaceholder = $("chatPlaceholder"); const textInput = $("textInput"); - const imageInput = $("imageInput"); - const audioInput = $("audioInput"); + const fileInput = $("fileInput"); const micBtn = $("micBtn"); const autoInputCheck = $("autoInputCheck"); const attachmentPreview = $("attachmentPreview"); @@ -17,7 +16,6 @@ const collapseBtn = $("collapseBtn"); const expandBtn = $("expandBtn"); const expandRail = $("expandRail"); - const sidebar = $("sidebar"); const appRoot = $("app"); const mainPanel = $("mainPanel"); const headerToggleBtn = $("headerToggleBtn"); @@ -39,10 +37,7 @@ /** @type {string[]} */ let sessions = []; /** @type {File[]} */ - let pendingImages = []; - /** @type {File | null} */ - let pendingAudio = null; - let sidebarCollapsed = false; + let pendingFiles = []; let sending = false; let mediaRecorder = null; let recorderStream = null; @@ -128,21 +123,32 @@ .replace(/"/g, """); } - function buildOptimisticUserContent(text, imageBlobUrls, audioPart) { + function buildOptimisticUserContent(text, fileParts) { const parts = []; - (imageBlobUrls || []).forEach((url) => { - parts.push({ type: "image", url }); + (fileParts || []).forEach((filePart) => { + if (filePart && filePart.kind === "image" && filePart.url) { + parts.push({ type: "image", url: filePart.url, name: filePart.name || "图片" }); + } else if (filePart && filePart.kind === "audio" && filePart.url) { + parts.push({ type: "audio", url: filePart.url, name: filePart.name || "音频" }); + } else if (filePart && filePart.name) { + parts.push({ type: "file", name: filePart.name }); + } }); const t = (text || "").trim(); if (t) parts.push({ type: "text", text: t }); - if (audioPart && audioPart.url) { - parts.push({ type: "audio", url: audioPart.url, name: audioPart.name || "音频" }); - } if (parts.length === 0) return null; if (parts.length === 1 && parts[0].type === "text") return parts[0].text; return parts; } + function inferUploadKind(file) { + const name = String(file?.name || "").toLowerCase(); + const type = String(file?.type || "").toLowerCase(); + if (type.startsWith("image/") || /\.(png|jpe?g|gif|webp|bmp|svg)$/.test(name)) return "image"; + if (type.startsWith("audio/") || /\.(wav|mp3|m4a|flac|ogg|webm)$/.test(name)) return "audio"; + return "file"; + } + function getTextFromContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; @@ -295,7 +301,7 @@ audioContext = new (window.AudioContext || window.webkitAudioContext)(); - if (audioContext.state === 'suspended') { + if (audioContext.state === "suspended") { await audioContext.resume(); } @@ -406,7 +412,7 @@ recorder.addEventListener("stop", () => { try { const blob = new Blob(recordedChunks, { type: recordingMimeType || recorder.mimeType || "audio/webm" }); - pendingAudio = buildRecordedAudioFile(blob, recordingMimeType || recorder.mimeType); + pendingFiles = pendingFiles.concat(buildRecordedAudioFile(blob, recordingMimeType || recorder.mimeType)); renderAttachmentChips(); resolve(); } catch (err) { @@ -627,10 +633,6 @@ return wrap; } - function parseAssistantContent(full) { - return renderMarkdownToFragment(full); - } - function fillAssistantBubbleImmediate(bubble, text) { const t = text == null ? "" : String(text); bubble.classList.add("msg-bubble--rich", "markdown-body"); @@ -895,6 +897,24 @@ return wrap; } + function renderDocumentBadge(part, options = {}) { + const wrap = document.createElement("div"); + wrap.className = "document-badge" + (options.role === "user" ? " document-badge--user" : ""); + + const icon = document.createElement("img"); + icon.className = "document-badge-icon"; + icon.src = "/image/文档.svg"; + icon.alt = "文档"; + + const name = document.createElement("span"); + name.className = "document-badge-name"; + name.textContent = part.name || "附件"; + + wrap.appendChild(icon); + wrap.appendChild(name); + return wrap; + } + function renderContentPart(part, options = {}) { if (typeof part === "string") { const div = document.createElement("div"); @@ -923,13 +943,8 @@ if (part.type === "audio" && part.url) { return renderAudioBubble(part, options); } - if (part.type === "file" && part.url) { - const a = document.createElement("a"); - a.href = part.url; - a.textContent = part.name || "附件"; - a.target = "_blank"; - a.rel = "noopener"; - return a; + if (part.type === "file") { + return renderDocumentBadge(part, options); } const div = document.createElement("div"); div.className = "msg-text"; @@ -994,27 +1009,16 @@ function renderAttachmentChips() { attachmentPreview.innerHTML = ""; - pendingImages.forEach((f, i) => { + pendingFiles.forEach((f, i) => { const chip = document.createElement("span"); chip.className = "chip"; chip.innerHTML = `${escapeHtml(f.name)} `; chip.querySelector("button").addEventListener("click", () => { - pendingImages = pendingImages.filter((_, j) => j !== i); + pendingFiles = pendingFiles.filter((_, j) => j !== i); renderAttachmentChips(); }); attachmentPreview.appendChild(chip); }); - if (pendingAudio) { - const chip = document.createElement("span"); - chip.className = "chip"; - chip.innerHTML = `${escapeHtml(pendingAudio.name)} `; - chip.querySelector("button").addEventListener("click", () => { - pendingAudio = null; - audioInput.value = ""; - renderAttachmentChips(); - }); - attachmentPreview.appendChild(chip); - } } async function bootstrap() { @@ -1039,10 +1043,8 @@ const data = await api("/api/sessions/new", { method: "POST" }); sessionId = data.sessionId; sessions = data.sessions || []; - pendingImages = []; - pendingAudio = null; - imageInput.value = ""; - audioInput.value = ""; + pendingFiles = []; + fileInput.value = ""; renderSessions(); renderAttachmentChips(); renderMessages(data.messages); @@ -1058,10 +1060,8 @@ const data = await api("/api/sessions/delete", { method: "POST", body: fd }); sessionId = data.sessionId; sessions = (data.sessions || []).filter((id) => id !== deletingSessionId); - pendingImages = []; - pendingAudio = null; - imageInput.value = ""; - audioInput.value = ""; + pendingFiles = []; + fileInput.value = ""; renderAttachmentChips(); renderSessions(); renderMessages(data.messages || []); @@ -1079,46 +1079,40 @@ async function sendChat() { if (!sessionId || sending) return; const trimmed = textInput.value.trim(); - if (!trimmed && pendingImages.length === 0 && !pendingAudio) return; + if (!trimmed && pendingFiles.length === 0) return; sending = true; sendBtn.disabled = true; const savedTextRaw = textInput.value; - const savedImages = pendingImages.slice(); - const savedAudio = pendingAudio; + const savedFiles = pendingFiles.slice(); const priorMessages = JSON.parse(JSON.stringify(lastRenderedMessages)); const blobUrls = []; - const hadOptimisticMedia = savedImages.length > 0 || !!savedAudio; + const hadOptimisticMedia = savedFiles.length > 0; if (hadOptimisticMedia) { - const imageBlobUrls = savedImages.map((f) => { - const u = URL.createObjectURL(f); - blobUrls.push(u); - return u; + const fileParts = savedFiles.map((f) => { + const kind = inferUploadKind(f); + if (kind === "image" || kind === "audio") { + const u = URL.createObjectURL(f); + blobUrls.push(u); + return { kind, url: u, name: f.name }; + } + return { kind, name: f.name }; }); - let audioPart = null; - if (savedAudio) { - const u = URL.createObjectURL(savedAudio); - blobUrls.push(u); - audioPart = { url: u, name: savedAudio.name }; - } - const optimisticContent = buildOptimisticUserContent(trimmed, imageBlobUrls, audioPart); + const optimisticContent = buildOptimisticUserContent(trimmed, fileParts); if (optimisticContent != null) { renderMessages([...priorMessages, { role: "user", content: optimisticContent }]); } textInput.value = ""; - pendingImages = []; - pendingAudio = null; - imageInput.value = ""; - audioInput.value = ""; + pendingFiles = []; + fileInput.value = ""; renderAttachmentChips(); } const fd = new FormData(); fd.append("session_id", sessionId); fd.append("text", trimmed); - savedImages.forEach((f) => fd.append("images", f, f.name)); - if (savedAudio) fd.append("audio", savedAudio, savedAudio.name); + savedFiles.forEach((f) => fd.append("files", f, f.name)); let typingEl = null; try { const startData = await api("/api/chat/start", { method: "POST", body: fd }); @@ -1127,10 +1121,8 @@ sessions = startData.sessions || []; if (!hadOptimisticMedia) { textInput.value = ""; - pendingImages = []; - pendingAudio = null; - imageInput.value = ""; - audioInput.value = ""; + pendingFiles = []; + fileInput.value = ""; renderAttachmentChips(); } renderSessions(); @@ -1164,8 +1156,7 @@ revokeBlobUrlStack(blobUrls); if (hadOptimisticMedia) { textInput.value = savedTextRaw; - pendingImages = savedImages.slice(); - pendingAudio = savedAudio; + pendingFiles = savedFiles.slice(); renderAttachmentChips(); renderMessages(priorMessages); } @@ -1177,18 +1168,10 @@ } } - imageInput.addEventListener("change", () => { - const files = Array.from(imageInput.files || []); - pendingImages = pendingImages.concat(files); - imageInput.value = ""; - renderAttachmentChips(); - textInput.focus(); - }); - - audioInput.addEventListener("change", () => { - const f = audioInput.files && audioInput.files[0]; - pendingAudio = f || null; - audioInput.value = ""; + fileInput.addEventListener("change", () => { + const files = Array.from(fileInput.files || []); + pendingFiles = pendingFiles.concat(files); + fileInput.value = ""; renderAttachmentChips(); textInput.focus(); }); diff --git a/src/web/image/文档.svg b/src/web/image/文档.svg new file mode 100644 index 0000000..2b4e61c --- /dev/null +++ b/src/web/image/文档.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/web/image/附件.svg b/src/web/image/附件.svg new file mode 100644 index 0000000..17b8ef1 --- /dev/null +++ b/src/web/image/附件.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/web/index.html b/src/web/index.html index 9a8d649..cd18cf1 100644 --- a/src/web/index.html +++ b/src/web/index.html @@ -81,7 +81,7 @@

开始对话

-

输入文字，或上传图片 / 音频 后点击发送。

+

输入文字，或上传附件 后点击发送。

支持用户音频输入、助手文本/语音/双模输出、会话本地媒体持久化。

@@ -101,13 +101,9 @@
- -