qwenasr-qwen36-voxcmp2 base model. -2026/05/14 night
This commit is contained in:
minorQAQ committed
commit
110fe78016
80 files changed
+6057
No files matched your search
Whitespace-only changes.
Binary file not shown.
Whitespace-only changes.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,178 @@
|
||||
from typing import Any, cast, Dict
|
||||
|
||||
from langchain_classic.agents import AgentExecutor, create_tool_calling_agent
|
||||
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
|
||||
from langchain_core.runnables import RunnablePassthrough, RunnableWithMessageHistory
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from agent.llm import Qwen36_35B_A3B_FP8_LLM
|
||||
from agent.tools import get_tools
|
||||
from agent.utils.agent_utils import assistant_text, get_history, normalize_user_text_content, summarize_history
|
||||
from agent.web.ui_session import (
|
||||
collect_media_paths_and_query_text,
|
||||
rewrite_user_message_files_to_session_dir,
|
||||
session_dir,
|
||||
sync_manifest_from_chat_history,
|
||||
)
|
||||
from agent.memory.system_prompt import MAIN_SYSTEM_PROMPT
|
||||
|
||||
# ---------- 1. 提示模板----------
|
||||
prompt = ChatPromptTemplate.from_messages([
|
||||
("system", "{system_message}"),
|
||||
("placeholder", "{chat_history}"),
|
||||
MessagesPlaceholder(variable_name="human_message"),
|
||||
("placeholder", "{agent_scratchpad}"),
|
||||
])
|
||||
|
||||
tools = get_tools()
|
||||
|
||||
# ---------- 2. 创建工具调用 agent ----------
|
||||
tool_calling_agent = create_tool_calling_agent(
|
||||
llm=Qwen36_35B_A3B_FP8_LLM.bind(tool_choice="auto"),
|
||||
tools=tools,
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
# ---------- 3. agent executor ----------
|
||||
agent_executor = AgentExecutor(
|
||||
agent=tool_calling_agent,
|
||||
tools=tools,
|
||||
verbose=True,
|
||||
handle_parsing_errors=True,
|
||||
stream_runnable=False,
|
||||
)
|
||||
|
||||
# ---------- 4. 带历史记录的链 ----------
|
||||
agent_with_history = RunnableWithMessageHistory(
|
||||
runnable=agent_executor,
|
||||
get_session_history=get_history,
|
||||
input_messages_key="human_message",
|
||||
history_messages_key="chat_history",
|
||||
)
|
||||
|
||||
# ---------- 5. 注入摘要和系统提示 ----------
|
||||
agent_with_history = (
|
||||
RunnablePassthrough.assign(
|
||||
# ✅ 从输入数据中的 agent_config 获取配置
|
||||
summarize_result=lambda x: summarize_history(x["input"], x["agent_config"])
|
||||
)
|
||||
| RunnablePassthrough.assign(
|
||||
input=lambda x: x["input"],
|
||||
chat_history=lambda x: x["summarize_result"]["chat_history"],
|
||||
system_message=lambda x: (
|
||||
MAIN_SYSTEM_PROMPT
|
||||
+ (
|
||||
"\n\n【此前对话摘要】\n" + x["summarize_result"]["summary"]
|
||||
if x["summarize_result"]["summary"]
|
||||
else ""
|
||||
)
|
||||
),
|
||||
human_message=lambda x: [HumanMessage(content=x["input"])],
|
||||
)
|
||||
| agent_with_history
|
||||
)
|
||||
|
||||
|
||||
# ---------- 6. 辅助函数：将前端 chat_history 转为 LangChain 消息列表 ----------
|
||||
def _extract_user_input_parts(user_msg: Dict[str, Any]) -> tuple[list[str], list[str], list[dict[str, Any]] | str]:
|
||||
image_paths, audio_paths, query_text = collect_media_paths_and_query_text(user_msg)
|
||||
|
||||
content_parts: list[dict[str, Any]] = []
|
||||
for img_path in image_paths:
|
||||
content_parts.append({"type": "image_url", "image_url": {"url": img_path}})
|
||||
|
||||
for audio_path in audio_paths:
|
||||
content_parts.append({
|
||||
"type": "audio",
|
||||
"audio_path": audio_path,
|
||||
})
|
||||
if query_text:
|
||||
content_parts.append({"type": "text", "text": query_text})
|
||||
|
||||
if content_parts:
|
||||
if len(content_parts) == 1 and content_parts[0].get("type") == "text":
|
||||
return image_paths, audio_paths, content_parts[0]["text"]
|
||||
return image_paths, audio_paths, content_parts
|
||||
return image_paths, audio_paths, normalize_user_text_content(user_msg.get("content", query_text))
|
||||
|
||||
# ---------- 7. 核心执行函数 ----------
|
||||
def execute_agent(
|
||||
chat_history: list[dict[str, Any]],
|
||||
session_id: str = "test",
|
||||
output_type: str = "text",
|
||||
) -> list[dict[str, Any]]:
|
||||
chat_history = chat_history or []
|
||||
|
||||
if not chat_history or chat_history[-1].get("role") != "user":
|
||||
return chat_history
|
||||
|
||||
if session_id:
|
||||
rewrite_user_message_files_to_session_dir(chat_history[-1], session_id)
|
||||
|
||||
_image_paths, _audio_paths, user_content = _extract_user_input_parts(chat_history[-1])
|
||||
|
||||
# 构建运行时配置
|
||||
config = {
|
||||
"configurable": {
|
||||
"session_id": session_id,
|
||||
},
|
||||
"metadata": {
|
||||
"output_type": output_type,
|
||||
"audio_output_dir": session_dir(session_id) if session_id else None,
|
||||
},
|
||||
"llm": Qwen36_35B_A3B_FP8_LLM,
|
||||
}
|
||||
|
||||
invoke_input = {
|
||||
"input": user_content,
|
||||
"agent_config": config,
|
||||
}
|
||||
|
||||
try:
|
||||
result = cast(dict[str, Any], agent_with_history.invoke(invoke_input, config))
|
||||
|
||||
text_content = assistant_text(result.get("output", ""))
|
||||
|
||||
output_type_from_config = config.get("metadata", {}).get("output_type", "text")
|
||||
audio_data = Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs or {}
|
||||
if audio_data.get("output_type") != output_type_from_config:
|
||||
audio_data = {}
|
||||
|
||||
assistant_meta = {
|
||||
"output_type": output_type_from_config,
|
||||
"autoplay": False,
|
||||
"audio": audio_data.get("audio"),
|
||||
}
|
||||
|
||||
if output_type_from_config == "audio" and assistant_meta.get("audio"):
|
||||
assistant_content = []
|
||||
elif output_type_from_config == "both" and assistant_meta.get("audio"):
|
||||
assistant_content = text_content
|
||||
else:
|
||||
assistant_content = text_content if text_content else "（空回复）"
|
||||
|
||||
chat_history.append({
|
||||
"role": "assistant",
|
||||
"content": assistant_content,
|
||||
"meta": assistant_meta,
|
||||
})
|
||||
|
||||
# 用完后清空，避免影响后续无音频的请求
|
||||
Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs = None
|
||||
|
||||
except Exception as e:
|
||||
# 异常处理保持不变
|
||||
import traceback
|
||||
print("[ERROR] 调用模型时出错:")
|
||||
traceback.print_exc()
|
||||
err_text = "抱歉，我遇到了一些问题，请稍后再试。"
|
||||
chat_history.append({"role": "assistant", "content": err_text})
|
||||
|
||||
if session_id:
|
||||
sync_manifest_from_chat_history(session_id, chat_history, "")
|
||||
|
||||
return chat_history
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 9.9 MiB |
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"version": 1,
|
||||
"turns": [
|
||||
{
|
||||
"user": {
|
||||
"parts": [
|
||||
{
|
||||
"kind": "image_url",
|
||||
"relpath": "20260514_222254/b50c460b07654d568e897d92c776675f.jpg"
|
||||
},
|
||||
{
|
||||
"kind": "audio",
|
||||
"relpath": "20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a"
|
||||
},
|
||||
{
|
||||
"kind": "text",
|
||||
"text": "请直接告诉我数量。"
|
||||
}
|
||||
],
|
||||
"image_description": ""
|
||||
},
|
||||
"assistant": {
|
||||
"text": "图片中有两只猫。",
|
||||
"output_type": "both",
|
||||
"autoplay": false,
|
||||
"audio_duration_seconds": 2.08,
|
||||
"audio_relpath": "20260514_222254/assistant_tts_222333_0.wav"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"version": 2,
|
||||
"turns": [
|
||||
{
|
||||
"user": {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "你好"
|
||||
}
|
||||
]
|
||||
},
|
||||
"assistant": {
|
||||
"text": "你好！很高兴为您服务。请问有什么我可以帮您的吗？",
|
||||
"output_type": "text",
|
||||
"autoplay": false,
|
||||
"audio_duration_seconds": null,
|
||||
"audio_relpath": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"user": {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "现在几点了"
|
||||
}
|
||||
]
|
||||
},
|
||||
"assistant": {
|
||||
"text": "现在是2026年5月15日 13:02。",
|
||||
"output_type": "text",
|
||||
"autoplay": false,
|
||||
"audio_duration_seconds": null,
|
||||
"audio_relpath": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import wave
|
||||
import pyaudio
|
||||
from contextlib import closing
|
||||
from typing import Any, List, Dict, Optional
|
||||
from pydantic import Field
|
||||
import re, json
|
||||
from datetime import datetime as dt
|
||||
|
||||
import requests
|
||||
from langchain_core.messages import HumanMessage, AIMessage, BaseMessage, SystemMessage
|
||||
from langchain_core.outputs import ChatResult, ChatGeneration
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
from agent.utils.image_utils import image_bytes_to_openai_image_url_part
|
||||
from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, ASR_TTS_BASE_URL
|
||||
from agent.utils.agent_utils import assistant_text, normalize_user_text_content
|
||||
from agent.utils.image_utils import (
|
||||
image_path_to_openai_image_url_part,
|
||||
image_bytes_to_openai_image_url_part,
|
||||
IMAGE_FILE_EXTENSIONS,
|
||||
)
|
||||
from agent.memory.system_prompt import VISION_SYSTEM_PROMPT
|
||||
|
||||
class ChatQwen(ChatOpenAI):
|
||||
output_type: str = Field(default="text")
|
||||
stream_play: bool = Field(default=False)
|
||||
last_audio_kwargs: Optional[dict] = Field(default=None)
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
super().__init__(
|
||||
model=kwargs.pop("model", LLM_MODEL),
|
||||
api_key=kwargs.pop("api_key", LLM_API_KEY),
|
||||
base_url=kwargs.pop("base_url", LLM_BASE_URL),
|
||||
timeout=kwargs.pop("timeout", LLM_TIMEOUT),
|
||||
max_retries=kwargs.pop("max_retries", 0),
|
||||
**kwargs
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def play_audio_bytes(audio_bytes: bytes, chunk_size: int = 1024) -> None:
|
||||
if not audio_bytes:
|
||||
return
|
||||
wf = wave.open(io.BytesIO(audio_bytes), 'rb')
|
||||
p = pyaudio.PyAudio()
|
||||
stream = p.open(
|
||||
format=p.get_format_from_width(wf.getsampwidth()),
|
||||
channels=wf.getnchannels(),
|
||||
rate=wf.getframerate(),
|
||||
output=True,
|
||||
)
|
||||
data = wf.readframes(chunk_size)
|
||||
while data:
|
||||
stream.write(data)
|
||||
data = wf.readframes(chunk_size)
|
||||
stream.stop_stream()
|
||||
stream.close()
|
||||
p.terminate()
|
||||
wf.close()
|
||||
|
||||
@staticmethod
|
||||
def _process_image_content(part: Dict[str, Any]) -> Dict[str, Any]:
|
||||
image_type = part.get("type")
|
||||
if image_type != "image_url":
|
||||
return part
|
||||
image_url = part.get("image_url", {})
|
||||
url = image_url.get("url", "")
|
||||
if url.startswith(("http://", "https://", "data:")):
|
||||
return part
|
||||
if isinstance(url, str) and os.path.exists(url):
|
||||
ext = os.path.splitext(url)[1].lower()
|
||||
if ext in IMAGE_FILE_EXTENSIONS:
|
||||
result = image_path_to_openai_image_url_part(url)
|
||||
return result if result else {"type": "text", "text": "[图片处理失败]"}
|
||||
if isinstance(url, bytes):
|
||||
result = image_bytes_to_openai_image_url_part(url)
|
||||
return result if result else {"type": "text", "text": "[图片处理失败]"}
|
||||
return part
|
||||
|
||||
@staticmethod
|
||||
def _convert_audio_to_wav(audio_path: str) -> str:
|
||||
ext = os.path.splitext(audio_path)[1].lower()
|
||||
if ext == ".wav":
|
||||
return audio_path
|
||||
try:
|
||||
from pydub import AudioSegment
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("当前 ASR 仅支持 WAV，但未安装 pydub/ffmpeg") from exc
|
||||
try:
|
||||
audio = AudioSegment.from_file(audio_path)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"无法读取音频文件: {exc}") from exc
|
||||
fd, wav_path = tempfile.mkstemp(suffix=".wav", prefix="asr_")
|
||||
os.close(fd)
|
||||
try:
|
||||
audio.export(wav_path, format="wav")
|
||||
return wav_path
|
||||
except Exception:
|
||||
if os.path.exists(wav_path):
|
||||
os.remove(wav_path)
|
||||
raise
|
||||
|
||||
def _audio_to_text(self, audio_path: str) -> str:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"音频文件不存在: {audio_path}")
|
||||
wav_path = self._convert_audio_to_wav(audio_path)
|
||||
cleanup_wav = wav_path != audio_path
|
||||
try:
|
||||
with open(wav_path, "rb") as f:
|
||||
resp = requests.post(
|
||||
f"{ASR_TTS_BASE_URL}/asr",
|
||||
files={"file": (os.path.basename(wav_path), f, "audio/wav")},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return (resp.json().get("text") or "").strip()
|
||||
finally:
|
||||
if cleanup_wav and os.path.exists(wav_path):
|
||||
os.remove(wav_path)
|
||||
|
||||
def _preprocess_messages(self, messages: List[BaseMessage]) -> List[BaseMessage]:
|
||||
new_messages = []
|
||||
for msg in messages:
|
||||
if isinstance(msg, HumanMessage) and isinstance(msg.content, str):
|
||||
text = msg.content.strip()
|
||||
if text.startswith("[") and text.endswith("]"):
|
||||
try:
|
||||
possible_list = json.loads(text)
|
||||
if isinstance(possible_list, list):
|
||||
msg = HumanMessage(content=possible_list, additional_kwargs=msg.additional_kwargs)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
if isinstance(msg, HumanMessage) and isinstance(msg.content, dict):
|
||||
msg = HumanMessage(content=normalize_user_text_content(msg.content), additional_kwargs=msg.additional_kwargs)
|
||||
elif isinstance(msg, HumanMessage) and isinstance(msg.content, list):
|
||||
new_content: List[Dict[str, Any] | str] = []
|
||||
for part in msg.content:
|
||||
if isinstance(part, dict):
|
||||
if part.get("type") == "audio":
|
||||
audio_path = part.get("audio_path")
|
||||
if audio_path:
|
||||
try:
|
||||
text = self._audio_to_text(audio_path)
|
||||
new_content.append({"type": "text", "text": text})
|
||||
except Exception as e:
|
||||
new_content.append({"type": "text", "text": f"[语音识别失败: {e}]"})
|
||||
elif part.get("type") == "image_url":
|
||||
new_content.append(self._process_image_content(part))
|
||||
elif part.get("type") == "text":
|
||||
text = (part.get("text") or "").strip()
|
||||
if text:
|
||||
new_content.append({"type": "text", "text": text})
|
||||
elif part.get("path") or (part.get("file") or {}).get("path"):
|
||||
file_path = part.get("path") or (part.get("file") or {}).get("path")
|
||||
if isinstance(file_path, str) and os.path.isfile(file_path):
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
if ext in IMAGE_FILE_EXTENSIONS:
|
||||
new_content.append(self._process_image_content({"type": "image_url", "image_url": {"url": file_path}}))
|
||||
else:
|
||||
new_content.append({"type": "text", "text": f"[用户上传了附件文件: {os.path.basename(file_path)}]"})
|
||||
else:
|
||||
text = normalize_user_text_content(part)
|
||||
if text:
|
||||
new_content.append({"type": "text", "text": text})
|
||||
elif isinstance(part, str):
|
||||
text = part.strip()
|
||||
if text:
|
||||
new_content.append(text)
|
||||
if len(new_content) == 1 and isinstance(new_content[0], str):
|
||||
msg = HumanMessage(content=new_content[0], additional_kwargs=msg.additional_kwargs)
|
||||
else:
|
||||
msg = HumanMessage(content=new_content, additional_kwargs=msg.additional_kwargs)
|
||||
new_messages.append(msg)
|
||||
return new_messages
|
||||
|
||||
def _text_to_audio(self, text: str) -> bytes:
|
||||
resp = requests.get(f"{ASR_TTS_BASE_URL}/tts", params={"text": text})
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
|
||||
@staticmethod
|
||||
def _get_wav_duration_seconds(audio_bytes: bytes) -> float | None:
|
||||
if not audio_bytes:
|
||||
return None
|
||||
try:
|
||||
with closing(wave.open(io.BytesIO(audio_bytes), "rb")) as wf:
|
||||
frame_rate = wf.getframerate()
|
||||
frame_count = wf.getnframes()
|
||||
if frame_rate <= 0:
|
||||
return None
|
||||
return round(frame_count / frame_rate, 2)
|
||||
except wave.Error:
|
||||
return None
|
||||
|
||||
def _apply_tts(self, result: ChatResult, output_type: str, audio_output_dir: str | None = None) -> ChatResult:
|
||||
if output_type not in ("audio", "both") or not audio_output_dir:
|
||||
return result
|
||||
os.makedirs(audio_output_dir, exist_ok=True)
|
||||
for idx, gen in enumerate(result.generations):
|
||||
if not isinstance(gen, ChatGeneration):
|
||||
continue
|
||||
msg = gen.message
|
||||
if not isinstance(msg, AIMessage):
|
||||
continue
|
||||
text = assistant_text(msg.content)
|
||||
if not text.strip():
|
||||
continue
|
||||
try:
|
||||
audio_bytes = self._text_to_audio(text)
|
||||
timestamp = dt.now().strftime("%H%M%S")
|
||||
audio_path = os.path.join(audio_output_dir, f"assistant_tts_{timestamp}_{idx}.wav")
|
||||
with open(audio_path, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
duration_seconds = self._get_wav_duration_seconds(audio_bytes)
|
||||
# 仅保存到实例属性，不再写入 additional_kwargs
|
||||
if idx == 0:
|
||||
self.last_audio_kwargs = {
|
||||
"audio": {
|
||||
"sample_rate": 24000,
|
||||
"format": "wav",
|
||||
"path": audio_path,
|
||||
"duration_seconds": duration_seconds,
|
||||
},
|
||||
"output_type": output_type,
|
||||
}
|
||||
if self.stream_play:
|
||||
self.play_audio_bytes(audio_bytes)
|
||||
if output_type == "audio":
|
||||
msg.content = ""
|
||||
except Exception as e:
|
||||
print(f"[TTS ERROR] 合成语音失败: {e}")
|
||||
# 发生错误时也清空 last_audio_kwargs，避免污染
|
||||
self.last_audio_kwargs = None
|
||||
return result
|
||||
|
||||
def _generate(
|
||||
self,
|
||||
messages: List[BaseMessage],
|
||||
stop: Optional[List[str]] = None,
|
||||
run_manager: Optional[Any] = None,
|
||||
**kwargs: Any,
|
||||
) -> ChatResult:
|
||||
metadata = {}
|
||||
if run_manager:
|
||||
metadata = getattr(run_manager, 'metadata', {})
|
||||
output_type = metadata.get("output_type", self.output_type)
|
||||
audio_output_dir = metadata.get("audio_output_dir")
|
||||
|
||||
# ✅ 移除自定义参数，防止它们传入 OpenAI API
|
||||
kwargs.pop("output_type", None)
|
||||
kwargs.pop("audio_output_dir", None)
|
||||
|
||||
messages = self._preprocess_messages(messages)
|
||||
|
||||
result = super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
|
||||
|
||||
if output_type in ("audio", "both"):
|
||||
result = self._apply_tts(result, output_type, audio_output_dir=audio_output_dir)
|
||||
if output_type == "audio":
|
||||
for gen in result.generations:
|
||||
if isinstance(gen, ChatGeneration) and isinstance(gen.message, AIMessage):
|
||||
gen.message.content = ""
|
||||
return result
|
||||
|
||||
def _create_chat_result(self, response, generation_info=None) -> ChatResult:
|
||||
result = super()._create_chat_result(response, generation_info)
|
||||
try:
|
||||
dumped = response if isinstance(response, dict) else response.model_dump(exclude={"choices": {"__all__": {"message": {"parsed"}}}})
|
||||
for i, choice in enumerate(dumped.get("choices") or []):
|
||||
if i >= len(result.generations):
|
||||
break
|
||||
lc_msg = result.generations[i].message
|
||||
msg = choice.get("message") or {}
|
||||
ak = lc_msg.additional_kwargs
|
||||
merged = ""
|
||||
for key in ("reasoning_content", "reasoning_details", "reasoning"):
|
||||
merged = assistant_text(msg.get(key))
|
||||
if merged:
|
||||
break
|
||||
if merged:
|
||||
ak.setdefault("reasoning_content", merged)
|
||||
raw_content = assistant_text(msg.get("content"))
|
||||
if not raw_content and merged:
|
||||
lc_msg.content = merged
|
||||
elif raw_content:
|
||||
lc_msg.content = raw_content
|
||||
content_text = lc_msg.content if isinstance(lc_msg.content, str) else ""
|
||||
pattern = re.compile(r'<tool_call>\s*(.*?)\s*</tool_call>', re.DOTALL)
|
||||
matches = pattern.findall(content_text)
|
||||
if matches:
|
||||
tool_calls = []
|
||||
for idx, match in enumerate(matches):
|
||||
try:
|
||||
call_data = json.loads(match)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
tool_calls.append({
|
||||
"id": f"call_{idx}",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": call_data.get("name", ""),
|
||||
"arguments": json.dumps(call_data.get("arguments", {}))
|
||||
}
|
||||
})
|
||||
if tool_calls:
|
||||
ak["tool_calls"] = tool_calls
|
||||
cleaned = pattern.sub("", content_text).strip()
|
||||
lc_msg.content = cleaned if cleaned else ""
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
def describe_image(self, image_path_or_url: str, prompt: str = "请详细描述这张图片的内容。") -> str:
|
||||
messages = [
|
||||
SystemMessage(content=[{"type": "text", "text": VISION_SYSTEM_PROMPT}]),
|
||||
HumanMessage(content=[
|
||||
{"type": "image_url", "image_url": {"url": image_path_or_url}},
|
||||
{"type": "text", "text": prompt}
|
||||
])
|
||||
]
|
||||
result = self.invoke(messages)
|
||||
return result.content
|
||||
|
||||
def describe_image_bytes(self, image_bytes: bytes, prompt: str = "请详细描述这张图片的内容。") -> str:
|
||||
img_part = image_bytes_to_openai_image_url_part(image_bytes)
|
||||
if img_part is None:
|
||||
return "图片处理失败"
|
||||
messages = [
|
||||
SystemMessage(content=[{"type": "text", "text": VISION_SYSTEM_PROMPT}]),
|
||||
HumanMessage(content=[img_part, {"type": "text", "text": prompt}])
|
||||
]
|
||||
result = self.invoke(messages)
|
||||
return result.content
|
||||
|
||||
|
||||
Qwen36_35B_A3B_FP8_LLM = ChatQwen()
|
||||
Whitespace-only changes.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,68 @@
|
||||
MAIN_SYSTEM_PROMPT = """你是一个多模态助手，能够直接理解用户上传的图片、语音等内容。
|
||||
【关键行为准则】
|
||||
- 你的默认行为是直接回答用户问题，不要调用任何工具。
|
||||
- 只有当某个信息是你**绝对无法从当前对话或上传内容中获取**，且该信息必须通过外部工具才能获得时，才允许调用工具。
|
||||
- 对于任何图片/音频输入，请始终**直接**利用你的视觉/听觉能力回答，**绝对不要**调用截图、OCR、日期查询等无关工具。
|
||||
- 如果你不确定是否需要工具，选择**不调用**，直接给出你能给出的最佳答案。
|
||||
- 违反上述规则是一种严重错误。
|
||||
"""
|
||||
|
||||
SUMMARY_SYSTEM_PROMPT ="""
|
||||
你是一个名为“Summary Agent”的专业摘要助手，专门负责对用户提供的文本内容进行高度凝练、准确的总结。
|
||||
|
||||
## 核心原则
|
||||
- 客观准确：严格基于原文内容，不添加个人观点、评论或外部知识。
|
||||
- 抓住要点：提取核心论点、关键数据、重要事件或结论，忽略次要细节、重复和示例。
|
||||
- 语言简洁：用清晰、简练的语句表达，避免冗长。
|
||||
- 保持原意：不歪曲原文事实和观点，即使原文存在错误或争议，仅如实概括。
|
||||
- 结构清晰：根据内容类型选择最合适的结构，如段落式或分点式，确保逻辑流畅。
|
||||
|
||||
## 输出格式
|
||||
- 默认以一段连贯的文字呈现摘要，必要时可使用编号或项目符号。
|
||||
- 摘要长度：如果用户未指定，则将原文字数压缩至 20%~30% 左右，或直接给出约3~5句话的精简摘要。对于长文，可适当增加长度。
|
||||
- 开头不需要“摘要：”等标记，直接输出摘要内容。（如用户要求标记，可加上）
|
||||
- 处理对话记录时，请总结：讨论主题、各方核心立场、达成的共识、存在的分歧及后续行动项。
|
||||
|
||||
## 特别行为
|
||||
- 若内容不足以形成有意义的摘要（如太短或无实际信息），请回复“内容过短，无法生成有效摘要”。
|
||||
- 若用户要求总结特定方面（如仅总结“原因”或“结论”），请针对性提取，并在摘要中注明聚焦点。
|
||||
- 当需要保留原文关键术语或专有名词时，可原样引用，并补充简要解释。
|
||||
- 多语言支持：用与原文相同的语言输出摘要，除非用户指定目标语言。
|
||||
|
||||
## 示例行为（模仿语气）
|
||||
用户：“请总结以下新闻：……（长新闻文本）……”
|
||||
你输出：“某公司于2026年5月发布了新一代量子芯片，性能较前代提升10倍，预计2027年量产。此举被业界视为量子计算商业化的关键突破，同时引发了对加密安全的讨论。”
|
||||
|
||||
现在，请以 Summary Agent 的身份，完成用户的摘要请求。
|
||||
"""
|
||||
|
||||
|
||||
VISION_SYSTEM_PROMPT = """
|
||||
你是专业的图像理解助手。
|
||||
首先请用客观、准确的中文描述用户提供的图片，覆盖主要物体、场景、人物动作、界面元素、图表数据与清晰可读的文字；
|
||||
涉及到你认识的具体品牌、时间、地点、人物、影视剧、物品等特定内容时请直接命名说明；
|
||||
如果用户提供了多张图片，请分别编号描述(如「图1」「图2」等)，便于后续问答引用。
|
||||
看不清的内容请如实说明，不要臆测。
|
||||
其次请认真回答用户的问题，结合图片内容进行分析推理，给出具体、简短、清晰、实用的回复；
|
||||
"""
|
||||
|
||||
DB_SYSTEM_PROMT = """
|
||||
你是一位敏捷的数据库分析助手，能够高效利用内置功能帮助用户从PostgreSQL数据库中获取答案。
|
||||
|
||||
给定一个数据库相关问题，你需要按照以下步骤操作：
|
||||
1.创建一个语法正确的PostgreSQL查询语句，优先使用SELECT或WITH语句，确保查询只读且安全。
|
||||
2.使用内置的SQL查询工具执行该查询，并获取结果。
|
||||
3.将查询结果总结为一个简洁、清晰的回答，突出关键数字和关系，帮助用户理解数据背后的含义。
|
||||
|
||||
【你的行动指南】
|
||||
1. **永远优先使用内置功能获取真实数据**。对于任何与数据库内容相关的问题，严禁编造答案。
|
||||
2. **智能判断调用顺序**：
|
||||
- 如果用户问题模糊（例如“帮我看看用户相关的数据”），先展示所有可用表，并询问用户具体需求。
|
||||
- 如果用户指明了具体表名，可以直接查看表结构，以便构思查询。
|
||||
- 如果用户直接提供了一条看似合理的SELECT或WITH语句，可以先验证其安全性，若通过则立即执行。
|
||||
3. **灵活处理查询结果**：当执行查询返回数据后，请将JSON结果归纳为人类友好的描述，突出关键数字和关系。若无数据返回，应如实告知并建议用户调整查询条件。
|
||||
4. **安全底线不可突破**：无论何时，只允许执行以SELECT或WITH开头的只读查询。若功能返回安全拦截错误，必须向用户解释原因并建议修改。
|
||||
5. **非数据库问题快速响应**：遇到问候、常识性问题或超出数据库范围的内容时，可以直接友好回应，不必强制调用功能。
|
||||
|
||||
请记住，你是一个协作伙伴，目标是让用户以最小的学习成本从数据库中获得有价值的信息。开始工作吧！
|
||||
"""
|
||||
@@ -0,0 +1,32 @@
|
||||
from .tool_main import *
|
||||
|
||||
# from .tool_database_class import *
|
||||
# from utils.database_utils import PostgreSQL_Manager
|
||||
|
||||
# 创建数据库管理器实例
|
||||
# connection = "postgresql+psycopg2://postgres:12345678@localhost:5432/agent_controller"
|
||||
# connection = "sqlite:///Agent/src/agent/history/data_base/chat_history.db"
|
||||
# _db_manager = PostgreSQL_Manager(connection)
|
||||
|
||||
# # 创建工具实例
|
||||
# _all_database_tools = [
|
||||
# Websearchclass(),
|
||||
# DateQueryClass(),
|
||||
# ListTableComment(db_manager=_db_manager),
|
||||
# ListTableSchema(db_manager=_db_manager),
|
||||
# SQLQuery(db_manager=_db_manager),
|
||||
# SQLQueryChecker(db_manager=_db_manager)
|
||||
# ]
|
||||
# def get_sql_tools():
|
||||
# return _all_database_tools
|
||||
|
||||
|
||||
_all_tools = [
|
||||
Websearchclass(),
|
||||
DateQueryClass(),
|
||||
ScreenCaptureDescribeClass(),
|
||||
BrowserControlClass()
|
||||
]
|
||||
|
||||
def get_tools():
|
||||
return _all_tools
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,54 @@
|
||||
from langchain.tools import tool
|
||||
|
||||
@tool("web_search")
|
||||
def web_search(query: str) -> str:
|
||||
"""
|
||||
一个用于执行网络搜索的工具。它接收一个查询语句作为输入，并返回 Google 的搜索结果。
|
||||
|
||||
Args:
|
||||
query (str): 搜索查询内容
|
||||
|
||||
Returns:
|
||||
str: 格式化后的搜索结果
|
||||
"""
|
||||
try:
|
||||
from langchain_community.utilities import GoogleSerperAPIWrapper
|
||||
# 这里直接使用了一个示例的 Serper API 密钥，实际使用时请替换为你自己的密钥
|
||||
SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5"
|
||||
# LangChain 官方封装的 Serper API 工具
|
||||
search = GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048)
|
||||
# 直接调用 run 方法获取结果，LangChain 已处理好格式
|
||||
return search.run(query)
|
||||
except Exception as e:
|
||||
return f"执行网络搜索时出错: {str(e)}"
|
||||
|
||||
@tool("date_query")
|
||||
def date_query() -> str:
|
||||
"""
|
||||
一个用于执行日期查询的工具。它接收一个查询语句作为输入，并返回相应的日期结果。
|
||||
|
||||
Returns:
|
||||
str: 格式化后的日期查询结果
|
||||
"""
|
||||
try:
|
||||
# 显示当前日期和时间
|
||||
from datetime import datetime
|
||||
now = datetime.now()
|
||||
return now.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
except Exception as e:
|
||||
return f"执行日期查询时出错: {str(e)}"
|
||||
|
||||
if __name__ == "__main__":
|
||||
# print(web_search.name) # 输出工具名称
|
||||
# print(web_search.description) # 输出工具描述
|
||||
# print(web_search.args_schema.model_json_schema()) # 输出工具参数结构
|
||||
# # 测试工具
|
||||
# query = "Python 中的列表和元组有什么区别？"
|
||||
# result = web_search.invoke({"query": query})
|
||||
# print(result)
|
||||
print(date_query.name) # 输出工具名称
|
||||
print(date_query.description) # 输出工具描述
|
||||
print(date_query.args_schema.model_json_schema()) # 输出工具参数结构
|
||||
print(date_query.invoke({})) # 测试工具
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
from langchain.tools import BaseTool
|
||||
from pydantic import Field, create_model, BaseModel
|
||||
from utils.database_utils import PostgreSQL_Manager
|
||||
import json
|
||||
from typing import Optional
|
||||
|
||||
class ListTableComment(BaseTool):
|
||||
|
||||
name: str = "list_table_comment"
|
||||
description: str = "一个列出数据库中所有表名和对应描述的工具，可以根据用户的查询返回相关的表名和描述列表。"
|
||||
db_manager : PostgreSQL_Manager
|
||||
|
||||
def _run(self, query: str) -> str:
|
||||
# 这里可以实现具体的列出数据库表的逻辑，例如调用数据库管理类的方法
|
||||
# 下面是一个示例实现，实际使用时请替换为你自己的数据库查询逻辑
|
||||
try:
|
||||
result_json = self.db_manager.get_table_comments()
|
||||
results = json.loads(result_json)
|
||||
if not isinstance(results, dict):
|
||||
raise ValueError("get_table_comments did not return a JSON object")
|
||||
result = f"数据库中共有 {len(results)} 张表"
|
||||
for i, (table_name, comment) in enumerate(results.items()):
|
||||
if not comment or comment.isspace():
|
||||
comment = "(暂无描述)"
|
||||
result += f"\n{i+1}. {table_name}: {comment}"
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"执行列出数据库表和描述时出错: {str(e)}"
|
||||
|
||||
async def _arun(self, query: str) -> str:
|
||||
return self._run(query)
|
||||
|
||||
class ListTableSchema(BaseTool):
|
||||
|
||||
name: str = "list_table_schema"
|
||||
description: str = "一个列出数据库中表名对应模式的工具，可以根据用户的查询返回相关的表名和模式信息，包括列定义、主键、外键等。输入应为逗号分隔的表名列表。"
|
||||
db_manager: PostgreSQL_Manager
|
||||
args_schema: type[BaseModel] = create_model("Input_type", table_names=(Optional[str], Field(description="要查询的表名列表,如果为None则返回所有表的模式信息,例如:'table1', 'table2'")))
|
||||
|
||||
def _run(self, table_names : Optional[str] = None) -> str:
|
||||
try:
|
||||
if table_names:
|
||||
table_list = [name.strip() for name in table_names.split(",")]
|
||||
result_json = self.db_manager.get_table_schema(table_list)
|
||||
results = json.loads(result_json)
|
||||
if not isinstance(results, dict):
|
||||
raise ValueError("get_table_schema did not return a JSON object")
|
||||
return json.dumps(results, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
return f"执行列出数据库表和模式时出错: {str(e)}"
|
||||
|
||||
async def _arun(self, table_names : Optional[str] = None) -> str:
|
||||
return self._run(table_names)
|
||||
|
||||
class SQLQuery(BaseTool):
|
||||
|
||||
name: str = "sql_query"
|
||||
description: str = "在PostgreSQL数据库中执行安全的SELECT或WITH查询并返回结果。"
|
||||
db_manager: PostgreSQL_Manager
|
||||
args_schema: type[BaseModel] = create_model("Input_type", query=(str, Field(description="要执行的SQL SELECT查询语句")))
|
||||
|
||||
def _run(self, query: str) -> str:
|
||||
try:
|
||||
result_json = self.db_manager.execute_query(query)
|
||||
results = json.loads(result_json)
|
||||
if isinstance(results, list):
|
||||
# 成功返回数据行
|
||||
return json.dumps(results, ensure_ascii=False, indent=2)
|
||||
elif isinstance(results, dict):
|
||||
if "error" in results:
|
||||
# 返回错误信息
|
||||
return f"查询执行失败: {results['error']}"
|
||||
elif "message" in results:
|
||||
# 查询成功但无数据返回
|
||||
return results["message"]
|
||||
else:
|
||||
# 其他字典格式的结果
|
||||
return json.dumps(results, ensure_ascii=False, indent=2)
|
||||
else:
|
||||
raise ValueError(f"execute_query 返回了意外的数据类型: {type(results)}")
|
||||
except Exception as e:
|
||||
return f"执行SQL查询时出错: {str(e)}"
|
||||
|
||||
async def _arun(self, query: str) -> str:
|
||||
return self._run(query)
|
||||
|
||||
class SQLQueryChecker(BaseTool):
|
||||
|
||||
name: str = "sql_query_checker"
|
||||
description: str = "一个检查SQL查询语句是否安全的工具，主要用于防止SQL注入攻击。输入为SQL查询语句，输出为检查结果。"
|
||||
args_schema: type[BaseModel] = create_model("Input_type", query=(str, Field(description="要检查的SQL查询语句")))
|
||||
db_manager: PostgreSQL_Manager
|
||||
|
||||
def _run(self, query: str) -> str:
|
||||
try:
|
||||
result_json = self.db_manager.validate_query(query)
|
||||
result = json.loads(result_json)
|
||||
if not isinstance(result, dict):
|
||||
raise ValueError("validate_query did not return a JSON object")
|
||||
return json.dumps(result, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
return f"执行SQL查询安全检查时出错: {str(e)}"
|
||||
async def _arun(self, query: str) -> str:
|
||||
return self._run(query)
|
||||
|
||||
if __name__ == "__main__":
|
||||
manager = PostgreSQL_Manager("postgresql+psycopg2://postgres:12345678@localhost:5432/agent_controller")
|
||||
list_table_tool = ListTableComment(db_manager=manager)
|
||||
print(list_table_tool.run("列出数据库中的所有表和描述"))
|
||||
# list_table_tool = ListTableSchema(db_manager=manager)
|
||||
# print(list_table_tool.run({"table_names": []}))
|
||||
# sql_query = SQLQuery(db_manager=manager)
|
||||
# print(sql_query.run("SELECT * FROM tasks;"))
|
||||
# sql_query_checker = SQLQueryChecker(db_manager=manager)
|
||||
# print("Valid query:")
|
||||
# print(sql_query_checker.run("SELECT * FROM tasks;"))
|
||||
# print("\nInvalid query:")
|
||||
# print(sql_query_checker.run("DROP TABLE tasks;"))
|
||||
@@ -0,0 +1,252 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from typing import Optional, Literal
|
||||
import sys
|
||||
import time
|
||||
import webbrowser
|
||||
import pyautogui
|
||||
|
||||
from langchain.tools import BaseTool
|
||||
from pydantic import BaseModel, Field, create_model
|
||||
|
||||
from agent.llm import Qwen36_35B_A3B_FP8_LLM
|
||||
from utils.env_utils import BROWSER_MAP
|
||||
|
||||
|
||||
class Websearchclass(BaseTool):
|
||||
|
||||
args_schema : type[BaseModel] = create_model("Input_type", query=(str, Field(description="搜索查询内容")))
|
||||
name : str = "web_search_class"
|
||||
description : str = "一个网络搜索工具，可以根据用户的查询返回相关的搜索结果。仅当需要查询近期知识或已有知识无法解答需要联网查询时调用，其余任何情况严禁调用。"
|
||||
|
||||
def _get_search_wrapper(self):
|
||||
from langchain_community.utilities import GoogleSerperAPIWrapper
|
||||
SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5"
|
||||
return GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048)
|
||||
|
||||
def _run(self, query: str) -> str:
|
||||
try:
|
||||
return self._get_search_wrapper().run(query)
|
||||
except Exception as e:
|
||||
return f"执行网络搜索时出错: {str(e)}"
|
||||
|
||||
async def _arun(self, query: str) -> str:
|
||||
try:
|
||||
return await self._get_search_wrapper().arun(query)
|
||||
except Exception as e:
|
||||
return f"执行网络搜索时出错: {str(e)}"
|
||||
|
||||
class DateQueryClass(BaseTool):
|
||||
|
||||
# 定义工具的输入参数结构，这里没有输入参数，所以是一个空的模型
|
||||
args_schema : type[BaseModel] = create_model("NoneInput")
|
||||
name : str = "date_query_class"
|
||||
description : str = "一个日期查询工具，可以返回当前的日期信息。仅当需要查询当前时间时调用，其余任何情况严禁调用。"
|
||||
|
||||
def _run(self) -> str:
|
||||
try:
|
||||
from datetime import datetime
|
||||
now = datetime.now()
|
||||
return now.strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception as e:
|
||||
return f"执行日期查询时出错: {str(e)}"
|
||||
|
||||
async def _arun(self) -> str:
|
||||
return self._run()
|
||||
|
||||
|
||||
class ScreenCaptureDescribeClass(BaseTool):
|
||||
"""截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。"""
|
||||
|
||||
args_schema: type[BaseModel] = create_model(
|
||||
"ScreenCaptureInput",
|
||||
focus_query=(
|
||||
Optional[str],
|
||||
Field(
|
||||
default=None,
|
||||
description="可选。希望重点观察的界面或问题，例如「错误提示全文」「浏览器地址栏」；留空则整体描述屏幕。",
|
||||
),
|
||||
),
|
||||
)
|
||||
name: str = "screen_capture_describe"
|
||||
description: str = (
|
||||
"截取当前主屏幕画面，并用视觉模型转为文字描述。"
|
||||
"注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。"
|
||||
)
|
||||
|
||||
def _run(self, focus_query: Optional[str] = None) -> str:
|
||||
# 1. 确保 pyautogui 可用
|
||||
try:
|
||||
import pyautogui
|
||||
except ImportError as e:
|
||||
return (
|
||||
f"未安装 pyautogui，无法截屏: {e}。"
|
||||
"请在项目环境中执行: pip install pyautogui"
|
||||
)
|
||||
|
||||
# 2. 截屏并转为 PNG 字节流
|
||||
try:
|
||||
pil_img = pyautogui.screenshot()
|
||||
buf = io.BytesIO()
|
||||
pil_img.save(buf, format="PNG")
|
||||
raw_png = buf.getvalue()
|
||||
except Exception as e:
|
||||
return f"截屏失败: {e}"
|
||||
|
||||
# 3. 构建描述指令
|
||||
query = (focus_query or "").strip()
|
||||
|
||||
# 4. 调用与对话图片上传完全相同的视觉模型进行描述
|
||||
try:
|
||||
# Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义
|
||||
text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query)
|
||||
except Exception as e:
|
||||
return f"屏幕图像视觉解析失败: {e}"
|
||||
|
||||
if not (text or "").strip():
|
||||
return "（已截屏，但模型未返回有效描述。）"
|
||||
return f"【当前屏幕视觉描述】\n{text.strip()}"
|
||||
|
||||
class BrowserControlClass(BaseTool):
|
||||
"""统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL"""
|
||||
|
||||
args_schema: type[BaseModel] = create_model(
|
||||
"BrowserControlInput",
|
||||
action=(Literal["open", "close"], Field(
|
||||
default="open",
|
||||
description="操作类型：'open' 打开/刷新，'close' 关闭"
|
||||
)),
|
||||
url=(Optional[str], Field(
|
||||
default=None,
|
||||
description="URL 地址、'current' 表示当前页，留空则操作整个浏览器窗口"
|
||||
)),
|
||||
browser=(Optional[str], Field(
|
||||
default=None,
|
||||
description="浏览器名称，如 'edge', 'chrome', 'firefox'，不指定则使用系统默认"
|
||||
)),
|
||||
)
|
||||
name: str = "browser_control"
|
||||
description: str = (
|
||||
"统一控制浏览器：打开/关闭浏览器、刷新/关闭标签页、打开指定 URL。\n"
|
||||
"- action='open', url=None: 打开浏览器首页\n"
|
||||
"- action='open', url='current': 刷新当前标签页\n"
|
||||
"- action='open', url='https://...': 在新标签页打开指定 URL\n"
|
||||
"- action='close', url=None: 关闭整个浏览器窗口\n"
|
||||
"- action='close', url='current': 关闭当前标签页\n"
|
||||
"- action='close', url='https://...': 关闭当前活动标签页"
|
||||
"仅用于进行浏览器操作时调用，其余任何情况严禁调用。"
|
||||
)
|
||||
|
||||
# ---------- 辅助方法 ----------
|
||||
def _get_browser_info(self, browser_name: Optional[str]):
|
||||
"""获取浏览器控制器和窗口标题关键字"""
|
||||
if browser_name is None:
|
||||
return webbrowser.get(), None
|
||||
|
||||
name_lower = browser_name.lower().strip()
|
||||
if name_lower not in BROWSER_MAP:
|
||||
raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}")
|
||||
|
||||
reg_name, title_keyword = BROWSER_MAP[name_lower]
|
||||
try:
|
||||
controller = webbrowser.get(reg_name)
|
||||
except webbrowser.Error:
|
||||
controller = webbrowser.get(browser_name)
|
||||
return controller, title_keyword
|
||||
|
||||
def _activate_window(self, title_keyword: str) -> bool:
|
||||
"""激活浏览器窗口（仅 Windows）"""
|
||||
if sys.platform != "win32":
|
||||
return False
|
||||
try:
|
||||
time.sleep(0.3)
|
||||
windows = pyautogui.getWindowsWithTitle(title_keyword)
|
||||
if windows:
|
||||
win = windows[0]
|
||||
if win.isMinimized:
|
||||
win.restore()
|
||||
win.activate()
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def _send_keys(self, *keys):
|
||||
"""发送快捷键，根据平台调整"""
|
||||
pyautogui.hotkey(*keys)
|
||||
|
||||
# ---------- 核心操作 ----------
|
||||
def _open_action(self, url: Optional[str], browser: Optional[str]):
|
||||
"""执行打开类操作"""
|
||||
controller, title_keyword = self._get_browser_info(browser)
|
||||
|
||||
if url is None:
|
||||
# 仅打开浏览器首页（激活或启动）
|
||||
controller.open("about:blank")
|
||||
if title_keyword:
|
||||
self._activate_window(title_keyword)
|
||||
browser_info = f" (使用 {browser})" if browser else ""
|
||||
return f"已打开浏览器首页{browser_info}"
|
||||
|
||||
elif url.lower() == "current":
|
||||
# 刷新当前页
|
||||
if title_keyword:
|
||||
if not self._activate_window(title_keyword):
|
||||
return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新"
|
||||
time.sleep(0.2)
|
||||
if sys.platform == "darwin":
|
||||
self._send_keys("command", "r")
|
||||
else:
|
||||
self._send_keys("f5")
|
||||
return "已刷新当前标签页"
|
||||
|
||||
else:
|
||||
# 打开指定 URL
|
||||
controller.open(url)
|
||||
if title_keyword:
|
||||
self._activate_window(title_keyword)
|
||||
return f"已尝试打开 URL: {url}"
|
||||
|
||||
def _close_action(self, url: Optional[str], browser: Optional[str]):
|
||||
"""执行关闭类操作"""
|
||||
_, title_keyword = self._get_browser_info(browser)
|
||||
|
||||
if url is None:
|
||||
# 关闭整个浏览器
|
||||
if title_keyword:
|
||||
if not self._activate_window(title_keyword):
|
||||
return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭"
|
||||
time.sleep(0.2)
|
||||
if sys.platform == "darwin":
|
||||
self._send_keys("command", "q")
|
||||
else:
|
||||
self._send_keys("alt", "f4")
|
||||
return "已尝试关闭浏览器窗口"
|
||||
|
||||
elif url.lower() == "current":
|
||||
# 关闭当前标签页
|
||||
if title_keyword:
|
||||
if not self._activate_window(title_keyword):
|
||||
return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页"
|
||||
time.sleep(0.2)
|
||||
if sys.platform == "darwin":
|
||||
self._send_keys("command", "w")
|
||||
else:
|
||||
self._send_keys("ctrl", "w")
|
||||
return "已尝试关闭当前标签页"
|
||||
|
||||
def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str:
|
||||
try:
|
||||
if action == "open":
|
||||
return self._open_action(url, browser)
|
||||
elif action == "close":
|
||||
return self._close_action(url, browser)
|
||||
else:
|
||||
return f"不支持的操作类型: {action}"
|
||||
except Exception as e:
|
||||
return f"执行浏览器操作时出错: {e}"
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 这里可以进行一些简单的测试，例如创建工具实例并调用方法
|
||||
pass
|
||||
Whitespace-only changes.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,181 @@
|
||||
# agent/agent_utils.py
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from langchain_community.chat_message_histories import SQLChatMessageHistory
|
||||
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
|
||||
|
||||
from agent.memory.system_prompt import SUMMARY_SYSTEM_PROMPT
|
||||
|
||||
# 关键改动：__file__ 在 agent/utils/agent_utils.py 下，
|
||||
# 取两次 dirname 得到 agent 目录（即 src/agent）
|
||||
_AGENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
HISTORY_ROOT = os.path.join(_AGENT_DIR, "history")
|
||||
_DB_DIR = os.path.join(HISTORY_ROOT, "data_base")
|
||||
CHAT_HISTORY_DB_FS = os.path.join(_DB_DIR, "chat_history.db")
|
||||
MULTI_INPUTS_ROOT = os.path.join(HISTORY_ROOT, "multi_inputs")
|
||||
|
||||
def ensure_dirs_and_migrate_db() -> None:
|
||||
os.makedirs(_DB_DIR, exist_ok=True)
|
||||
os.makedirs(os.path.join(HISTORY_ROOT, "multi_inputs"), exist_ok=True)
|
||||
old_db = os.path.join(HISTORY_ROOT, "chat_history.db")
|
||||
if os.path.isfile(old_db) and not os.path.isfile(CHAT_HISTORY_DB_FS):
|
||||
shutil.move(old_db, CHAT_HISTORY_DB_FS)
|
||||
|
||||
ensure_dirs_and_migrate_db()
|
||||
|
||||
def get_history(session_id: str) -> SQLChatMessageHistory:
|
||||
return SQLChatMessageHistory(
|
||||
session_id=session_id,
|
||||
connection="sqlite:///" + CHAT_HISTORY_DB_FS.replace("\\", "/"),
|
||||
)
|
||||
|
||||
def summarize_history(query, config):
|
||||
"""
|
||||
剪辑最近8条消息，并对更早的消息生成摘要。
|
||||
返回 {'summary': str|None, 'chat_history': list[BaseMessage]}
|
||||
"""
|
||||
session_id = config["configurable"]["session_id"]
|
||||
if not session_id:
|
||||
raise ValueError("Session ID is required in the config.")
|
||||
chat_history = get_history(session_id)
|
||||
stored_messages = chat_history.messages
|
||||
|
||||
if len(stored_messages) <= 8:
|
||||
return {"summary": None, "chat_history": stored_messages}
|
||||
|
||||
recent = stored_messages[-8:]
|
||||
old = stored_messages[:-8]
|
||||
|
||||
summary_prompt = ChatPromptTemplate.from_messages([
|
||||
("system", SUMMARY_SYSTEM_PROMPT),
|
||||
MessagesPlaceholder(variable_name="messages_to_summarize", optional=True),
|
||||
("human", "请生成包含上述对话核心内容的摘要，保留重要事实和决策。"),
|
||||
])
|
||||
|
||||
llm = config.get("llm")
|
||||
chain = summary_prompt | llm
|
||||
summary = chain.invoke(
|
||||
{"messages_to_summarize": old},
|
||||
config={
|
||||
"metadata": {
|
||||
"output_type": "text",
|
||||
"audio_output_dir": None,
|
||||
}
|
||||
}
|
||||
)
|
||||
return {"summary": summary.content, "chat_history": recent}
|
||||
|
||||
|
||||
def assistant_text(value) -> str:
|
||||
"""把网关返回的 content、reasoning 等（str 或块列表）收成一段可见文本。"""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, list):
|
||||
parts: list[str] = []
|
||||
for item in value:
|
||||
if isinstance(item, str) and item.strip():
|
||||
parts.append(item.strip())
|
||||
elif isinstance(item, dict):
|
||||
for key in ("text", "reasoning_content", "content"):
|
||||
v = item.get(key)
|
||||
if isinstance(v, str) and v.strip():
|
||||
parts.append(v.strip())
|
||||
break
|
||||
return "\n".join(parts).strip()
|
||||
if isinstance(value, dict):
|
||||
for key in ("text", "content", "input", "output", "query", "answer", "result"):
|
||||
v = value.get(key)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
except TypeError:
|
||||
return str(value).strip()
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def normalize_user_text_content(value) -> str:
|
||||
"""尽量把用户输入收敛为纯文本，避免把 dict/list 结构直接送进主模型。"""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
if isinstance(value, dict):
|
||||
for key in ("text", "content", "input", "query", "question", "message"):
|
||||
v = value.get(key)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
content = value.get("content")
|
||||
if content is not None:
|
||||
text = normalize_user_text_content(content)
|
||||
if text:
|
||||
return text
|
||||
return assistant_text(value)
|
||||
if isinstance(value, list):
|
||||
parts: list[str] = []
|
||||
for item in value:
|
||||
if isinstance(item, str) and item.strip():
|
||||
parts.append(item.strip())
|
||||
elif isinstance(item, dict):
|
||||
if item.get("type") == "text":
|
||||
text = (item.get("text") or "").strip()
|
||||
if text:
|
||||
parts.append(text)
|
||||
else:
|
||||
text = normalize_user_text_content(item)
|
||||
if text:
|
||||
parts.append(text)
|
||||
return "\n".join(parts).strip()
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def extract_result_from_invoke(value) -> str:
|
||||
"""兼容不同 invoke 返回结构，尽量提取可显示的助手文本。"""
|
||||
if value is None:
|
||||
return ""
|
||||
|
||||
if hasattr(value, "content"):
|
||||
text = assistant_text(getattr(value, "content", None))
|
||||
if text:
|
||||
return text
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key in ("output", "content", "text", "answer", "result"):
|
||||
text = assistant_text(value.get(key))
|
||||
if text:
|
||||
return text
|
||||
|
||||
message = value.get("message")
|
||||
if message is not None:
|
||||
text = extract_result_from_invoke(message)
|
||||
if text:
|
||||
return text
|
||||
|
||||
generations = value.get("generations")
|
||||
if isinstance(generations, list):
|
||||
for generation in generations:
|
||||
text = extract_result_from_invoke(generation)
|
||||
if text:
|
||||
return text
|
||||
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
text = extract_result_from_invoke(item)
|
||||
if text:
|
||||
return text
|
||||
|
||||
if hasattr(value, "message"):
|
||||
text = extract_result_from_invoke(getattr(value, "message", None))
|
||||
if text:
|
||||
return text
|
||||
|
||||
if hasattr(value, "additional_kwargs"):
|
||||
text = assistant_text(getattr(value, "additional_kwargs", {}).get("reasoning_content"))
|
||||
if text:
|
||||
return text
|
||||
|
||||
return ""
|
||||
@@ -0,0 +1,255 @@
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
import json
|
||||
from typing import Optional, List
|
||||
from uuid import UUID
|
||||
from datetime import datetime
|
||||
|
||||
class JSONEncoder(json.JSONEncoder):
|
||||
"""自定义JSON编码器，处理非标准序列化类型"""
|
||||
def default(self, obj):
|
||||
if isinstance(obj, UUID):
|
||||
return str(obj)
|
||||
elif isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
elif hasattr(obj, '__dict__'):
|
||||
return str(obj)
|
||||
return super().default(obj)
|
||||
|
||||
def make_json_serializable(obj):
|
||||
"""递归转换对象为JSON序列化格式"""
|
||||
if isinstance(obj, dict):
|
||||
return {k: make_json_serializable(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, (list, tuple)):
|
||||
return [make_json_serializable(item) for item in obj]
|
||||
elif isinstance(obj, UUID):
|
||||
return str(obj)
|
||||
elif isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
elif hasattr(obj, '__dict__'):
|
||||
return str(obj)
|
||||
else:
|
||||
return obj
|
||||
|
||||
class PostgreSQL_Manager:
|
||||
def __init__(self, connection_params):
|
||||
"""
|
||||
PostgreSQL数据库管理类,提供连接和操作数据库的功能
|
||||
Args:
|
||||
connection_params (dict): 连接参数字符串，格式为：
|
||||
postgresql+psycopg2://用户名:密码@主机:端口/数据库名
|
||||
本机默认为:postgresql+psycopg2://postgres:12345678@localhost:5432/agent_controller
|
||||
"""
|
||||
self.engine = create_engine(connection_params, pool_size=5, pool_recycle=7200)
|
||||
|
||||
def get_table_names(self) -> list[str]:
|
||||
"""
|
||||
获取数据库中的所有表名
|
||||
Returns:
|
||||
list: 数据库中的表名列表
|
||||
"""
|
||||
try:
|
||||
# 使用SQLAlchemy的inspect功能获取数据库表名
|
||||
inspector = inspect(self.engine)
|
||||
return inspector.get_table_names()
|
||||
except Exception as e:
|
||||
print(f"Error occurred while fetching table names: {e}")
|
||||
return []
|
||||
|
||||
def get_table_comments(self, table_name: str = None) -> str:
|
||||
"""
|
||||
获取指定表/所有表的注释信息,并整理成JSON格式
|
||||
Args:
|
||||
table_name (str, optional): 表名,如果为None则返回所有表
|
||||
Returns:
|
||||
str: JSON格式的表的注释信息
|
||||
"""
|
||||
try:
|
||||
inspector = inspect(self.engine)
|
||||
if table_name is None: # 返回所有表的注释
|
||||
table_names = inspector.get_table_names()
|
||||
comments_dict = {table: inspector.get_table_comment(table).get('text', '') for table in table_names}
|
||||
serializable_dict = make_json_serializable(comments_dict)
|
||||
return json.dumps(serializable_dict, ensure_ascii=False, indent=2, cls=JSONEncoder)
|
||||
else:
|
||||
comment = inspector.get_table_comment(table_name).get('text', '')
|
||||
result = {"table_name": table_name, "comment": comment}
|
||||
serializable_result = make_json_serializable(result)
|
||||
return json.dumps(serializable_result, ensure_ascii=False, indent=2, cls=JSONEncoder)
|
||||
except Exception as e:
|
||||
error_msg = {"error": f"Error occurred while fetching table comments: {str(e)}"}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
|
||||
def get_table_schema(self, table_names: Optional[List[str]] = None) -> str:
|
||||
"""
|
||||
获取指定表/所有表的schema信息,并整理成JSON格式
|
||||
Args:
|
||||
table_names (List[str], optional): 表名列表,如果为None则返回所有表
|
||||
Returns:
|
||||
str: JSON格式的表的schema信息
|
||||
"""
|
||||
try:
|
||||
inspector = inspect(self.engine)
|
||||
table_schema = {}
|
||||
names_to_query = table_names if table_names else inspector.get_table_names()
|
||||
for table_name in names_to_query:
|
||||
columns = inspector.get_columns(table_name)
|
||||
pk_constraints = inspector.get_pk_constraint(table_name)
|
||||
primary_keys = pk_constraints['constrained_columns'] if pk_constraints else []
|
||||
foreign_keys = inspector.get_foreign_keys(table_name)
|
||||
indexes = inspector.get_indexes(table_name)
|
||||
table_schema[table_name] = {
|
||||
'columns': columns,
|
||||
'primary_keys': primary_keys,
|
||||
'foreign_keys': foreign_keys,
|
||||
'indexes': indexes
|
||||
}
|
||||
# 转换为JSON序列化格式
|
||||
serializable_schema = make_json_serializable(table_schema)
|
||||
return json.dumps(serializable_schema, ensure_ascii=False, indent=2, cls=JSONEncoder)
|
||||
except Exception as e:
|
||||
error_msg = {"error": f"Error occurred while fetching table schema: {str(e)}"}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
|
||||
def execute_query(self, query: str) -> str:
|
||||
"""
|
||||
执行任意SQL查询,并返回结果
|
||||
Args:
|
||||
query (str): SQL查询语句
|
||||
Returns:
|
||||
str: JSON格式的查询结果或错误信息
|
||||
"""
|
||||
# 安全检查，防止执行危险的SQL语句，仅允许select和with查询语句
|
||||
if not query.strip().lower().startswith(('select', 'with')):
|
||||
error_msg = {"error": "Only SELECT and WITH queries are allowed."}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
|
||||
try:
|
||||
with self.engine.connect() as conn:
|
||||
result = conn.execute(text(query))
|
||||
# 获取列名
|
||||
columns = result.keys()
|
||||
# 获取数据（限制返回数据量防止内存溢出）
|
||||
rows = result.fetchmany(100) # 只获取前100行数据
|
||||
if not rows:
|
||||
return json.dumps({"message": "Query executed successfully, no rows returned."}, ensure_ascii=False, indent=2)
|
||||
else:
|
||||
result_data = []
|
||||
for row in rows:
|
||||
row_dict = dict(zip(columns, row))
|
||||
result_data.append(row_dict)
|
||||
# 转换为JSON序列化格式
|
||||
serializable_data = make_json_serializable(result_data)
|
||||
return json.dumps(serializable_data, ensure_ascii=False, indent=2, cls=JSONEncoder)
|
||||
except Exception as e:
|
||||
error_msg = {"error": f"Error occurred while executing query: {str(e)}"}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
|
||||
def validate_query(self, query: str) -> str:
|
||||
"""
|
||||
验证SQL查询语句的合法性
|
||||
Args:
|
||||
query (str): SQL查询语句
|
||||
Returns:
|
||||
bool: 查询语句是否合法
|
||||
"""
|
||||
if not query or not query.strip():
|
||||
error_msg = {"error": f"Error occurred while validating query: Query is must not be empty."}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
|
||||
# 简单的安全检查，防止执行危险的SQL语句，仅允许select和with查询语句
|
||||
if not query.strip().lower().startswith(('select', 'with')):
|
||||
error_msg = {"error": f"Error occurred while validating query: Only SELECT and WITH queries are allowed."}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
|
||||
try:
|
||||
with self.engine.connect() as conn:
|
||||
if self.engine.dialect.name == 'postgresql':
|
||||
# 使用EXPLAIN分析查询计划来验证查询的合法性
|
||||
explain_query = f"EXPLAIN {query}"
|
||||
else:
|
||||
# 对于其他数据库，可以直接执行查询但不获取结果来验证合法性
|
||||
explain_query = query
|
||||
conn.execute(text(explain_query))
|
||||
return json.dumps({"message": "Query is valid."}, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
error_msg = {"error": f"Error occurred while validating query: {str(e)}"}
|
||||
return json.dumps(error_msg, ensure_ascii=False, indent=2)
|
||||
# def rename_table(self, old_table_name: str, new_table_name: str) -> bool:
|
||||
# """
|
||||
# 重命名数据库表
|
||||
# Args:
|
||||
# old_table_name (str): 原表名
|
||||
# new_table_name (str): 新表名
|
||||
# Returns:
|
||||
# bool: 操作是否成功
|
||||
# """
|
||||
# try:
|
||||
# with self.engine.connect() as conn:
|
||||
# sql = text(f"ALTER TABLE {old_table_name} RENAME TO {new_table_name}")
|
||||
# conn.execute(sql)
|
||||
# conn.commit()
|
||||
# print(f"Table '{old_table_name}' renamed to '{new_table_name}' successfully.")
|
||||
# return True
|
||||
# except Exception as e:
|
||||
# print(f"Error occurred while renaming table: {e}")
|
||||
# return False
|
||||
|
||||
# def update_table_comment(self, table_name: str, comment: str) -> bool:
|
||||
# """
|
||||
# 更新表的注释
|
||||
# Args:
|
||||
# table_name (str): 表名
|
||||
# comment (str): 新的注释内容
|
||||
# Returns:
|
||||
# bool: 操作是否成功
|
||||
# """
|
||||
# try:
|
||||
# with self.engine.connect() as conn:
|
||||
# sql = text(f"COMMENT ON TABLE {table_name} IS '{comment}'")
|
||||
# conn.execute(sql)
|
||||
# conn.commit()
|
||||
# print(f"Comment for table '{table_name}' updated successfully.")
|
||||
# return True
|
||||
# except Exception as e:
|
||||
# print(f"Error occurred while updating table comment: {e}")
|
||||
# return False
|
||||
if __name__ == "__main__":
|
||||
DB_CONFIG = {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"database": "agent_controller",
|
||||
"user": "postgres",
|
||||
"password": "12345678"
|
||||
}
|
||||
connection_params = f"postgresql+psycopg2://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}?client_encoding=utf8"
|
||||
db_manager = PostgreSQL_Manager(connection_params)
|
||||
|
||||
# 获取并打印所有表名和注释（JSON格式）
|
||||
print("All table comments (JSON):")
|
||||
all_comments_json = db_manager.get_table_comments()
|
||||
print(all_comments_json)
|
||||
|
||||
# 示例：获取特定表的注释（JSON格式）
|
||||
# table_names = db_manager.get_table_names()
|
||||
# if table_names:
|
||||
# print(f"\nSpecific table comment for '{table_names[0]}' (JSON):")
|
||||
# specific_comment_json = db_manager.get_table_comments(table_names[0])
|
||||
# print(specific_comment_json)
|
||||
|
||||
# table_name = "result_artifacts"
|
||||
# new_comment = "用于存放任务运行结果的表"
|
||||
# db_manager.update_table_comment(table_name, new_comment)
|
||||
|
||||
# table_name = "agent_state_history"
|
||||
# new_comment = "用于存放agent状态历史的表"
|
||||
# db_manager.update_table_comment(table_name, new_comment)
|
||||
|
||||
# table_name = "agents"
|
||||
# new_comment = "用于存放agent的注册表"
|
||||
# db_manager.update_table_comment(table_name, new_comment)
|
||||
|
||||
# table_name = "tasks"
|
||||
# new_comment = "用于存放任务的仓库表"
|
||||
# db_manager.update_table_comment(table_name, new_comment)
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
from typing import Dict, Tuple
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv(override=True)
|
||||
|
||||
|
||||
def _get_env(name: str, default: str | None = None, cast: type | None = None, required: bool = False):
|
||||
value = os.getenv(name, default)
|
||||
if value is None and required:
|
||||
raise EnvironmentError(f"Missing required environment variable: {name}")
|
||||
if cast is None or value is None:
|
||||
return value
|
||||
try:
|
||||
return cast(value)
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise ValueError(f"Invalid value for {name}: {value}") from exc
|
||||
|
||||
|
||||
LLM_BASE_URL = _get_env("LLM_BASE_URL", required=True)
|
||||
LLM_API_KEY = _get_env("LLM_API_KEY", required=True)
|
||||
LLM_MODEL = _get_env("LLM_MODEL", required=True)
|
||||
LLM_TIMEOUT = _get_env("LLM_TIMEOUT", default="60", cast=float)
|
||||
LLM_REQUESTS_PER_SECOND = _get_env("LLM_REQUESTS_PER_SECOND", default="10", cast=float)
|
||||
LLM_MAX_BUCKET_SIZE = _get_env("LLM_MAX_BUCKET_SIZE", default="10", cast=int)
|
||||
LLM_CHECK_EVERY_N_SECONDS = _get_env("LLM_CHECK_EVERY_N_SECONDS", default="0.1", cast=float)
|
||||
ASR_TTS_BASE_URL = _get_env("ASR_TTS_BASE_URL")
|
||||
|
||||
BROWSER_MAP: Dict[str, Tuple[str, str]] = {
|
||||
"edge": ("microsoft-edge", "Microsoft Edge"),
|
||||
"chrome": ("chrome", "Google Chrome"),
|
||||
"firefox": ("firefox", "Mozilla Firefox"),
|
||||
"safari": ("safari", "Safari"),
|
||||
"opera": ("opera", "Opera"),
|
||||
"brave": ("brave", "brave browser"),
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
"""图片缩放、体积控制与 OpenAI 风格多模态 image_url 片段（供 LLM / Agent 使用）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
from typing import Any, Protocol
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class SupportsChatCompletion(Protocol):
|
||||
"""具备 ``chat_completion(messages, **kwargs)`` 的 LLM（如 ``ChatQwen35OpenAI``）。"""
|
||||
|
||||
def chat_completion(self, messages: list[Any], **kwargs: Any) -> Any: ...
|
||||
|
||||
# 小写扩展名（含点），用于判断路径是否为支持的栅格图
|
||||
IMAGE_FILE_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp"})
|
||||
|
||||
|
||||
def guess_image_mime(extension_without_dot: str) -> str:
|
||||
"""根据扩展名返回 image/* MIME，未知则回退为 image/jpeg。"""
|
||||
ext = extension_without_dot.lower().lstrip(".")
|
||||
if ext in ("png", "jpg", "jpeg", "gif", "bmp", "webp"):
|
||||
return "image/jpeg" if ext == "jpg" else f"image/{ext}"
|
||||
return "image/jpeg"
|
||||
|
||||
|
||||
def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = 768) -> bytes:
|
||||
"""
|
||||
控制图片体积，避免 data-url base64 触发超长 token。
|
||||
- 若图片尺寸过大，等比例缩放（默认最大边 max_size）
|
||||
- 优先压成 JPEG（透明图保底转白底 RGB）
|
||||
"""
|
||||
if isinstance(image_path_or_bytes, str):
|
||||
img = Image.open(image_path_or_bytes)
|
||||
original_format = img.format or "PNG"
|
||||
else:
|
||||
img = Image.open(io.BytesIO(image_path_or_bytes))
|
||||
original_format = img.format or "PNG"
|
||||
|
||||
if img.mode == "RGBA":
|
||||
background = Image.new("RGB", img.size, (255, 255, 255))
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
original_format = "JPEG"
|
||||
elif img.mode == "LA":
|
||||
background = Image.new("L", img.size, 255)
|
||||
background.paste(img, mask=img.split()[-1])
|
||||
img = background
|
||||
|
||||
original_width, original_height = img.size
|
||||
|
||||
if original_width > max_size or original_height > max_size:
|
||||
ratio = min(max_size / original_width, max_size / original_height)
|
||||
new_width = int(original_width * ratio)
|
||||
new_height = int(original_height * ratio)
|
||||
img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
|
||||
print(f"图片已从 {original_width}x{original_height} 缩放到 {new_width}x{new_height}")
|
||||
else:
|
||||
print(f"图片尺寸 {original_width}x{original_height} 未超过限制，无需缩放")
|
||||
|
||||
img_buffer = io.BytesIO()
|
||||
save_format = (original_format or "PNG").upper()
|
||||
if save_format not in ("PNG", "JPEG", "JPG"):
|
||||
save_format = "JPEG"
|
||||
|
||||
if save_format in ("JPEG", "JPG"):
|
||||
if img.mode not in ("RGB", "L"):
|
||||
img = img.convert("RGB")
|
||||
img.save(
|
||||
img_buffer,
|
||||
format="JPEG",
|
||||
quality=70,
|
||||
optimize=True,
|
||||
progressive=True,
|
||||
)
|
||||
else:
|
||||
img.save(img_buffer, format="PNG", optimize=True)
|
||||
img_buffer.seek(0)
|
||||
return img_buffer.getvalue()
|
||||
|
||||
|
||||
def image_to_base64(image_content: str | bytes) -> str | None:
|
||||
"""将图片路径或原始字节转为 base64 字符串；失败返回 None。"""
|
||||
try:
|
||||
processed = resize_image_if_needed(image_content)
|
||||
return base64.b64encode(processed).decode("utf-8")
|
||||
except Exception as e:
|
||||
print(f"处理图片 {image_content!r} 时出错: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _mime_from_magic(data: bytes) -> str:
|
||||
"""根据压缩后图片文件头推断 MIME，用于内存字节构造 data URL。"""
|
||||
if len(data) >= 2 and data[:2] == b"\xff\xd8":
|
||||
return "image/jpeg"
|
||||
if len(data) >= 8 and data[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
return "image/png"
|
||||
if len(data) >= 6 and data[:6] in (b"GIF87a", b"GIF89a"):
|
||||
return "image/gif"
|
||||
return "image/jpeg"
|
||||
|
||||
|
||||
def image_bytes_to_openai_image_url_part(
|
||||
image_bytes: bytes,
|
||||
*,
|
||||
detail: str | None = None,
|
||||
) -> dict | None:
|
||||
"""将内存中的图片字节（如截屏 PNG）转为 OpenAI 风格的 image_url 块；失败返回 None。"""
|
||||
try:
|
||||
processed = resize_image_if_needed(image_bytes)
|
||||
except Exception as e:
|
||||
print(f"处理图片字节时出错: {e}")
|
||||
return None
|
||||
mime = _mime_from_magic(processed)
|
||||
b64_str = base64.b64encode(processed).decode("utf-8")
|
||||
url = f"data:{mime};base64,{b64_str}"
|
||||
image_url: dict = {"url": url}
|
||||
if detail:
|
||||
image_url["detail"] = detail
|
||||
return {"type": "image_url", "image_url": image_url}
|
||||
|
||||
|
||||
def image_path_to_openai_image_url_part(
|
||||
file_path: str,
|
||||
*,
|
||||
detail: str | None = None,
|
||||
) -> dict | None:
|
||||
"""
|
||||
从本地图片路径构造 OpenAI Chat Completions 的 image_url 内容块。
|
||||
无法编码时返回 None。
|
||||
"""
|
||||
b64_str = image_to_base64(file_path)
|
||||
if not b64_str:
|
||||
return None
|
||||
ext = os.path.splitext(file_path)[1].lower().lstrip(".")
|
||||
mime_type = guess_image_mime(ext)
|
||||
url = f"data:{mime_type};base64,{b64_str}"
|
||||
image_url: dict = {"url": url}
|
||||
if detail:
|
||||
image_url["detail"] = detail
|
||||
return {"type": "image_url", "image_url": image_url}
|
||||
@@ -0,0 +1,38 @@
|
||||
import sys
|
||||
from loguru import logger
|
||||
|
||||
# Trace < Debug < Info < Success < Warning < Error < Critical
|
||||
|
||||
|
||||
class MyLogger:
|
||||
def __init__(self):
|
||||
self.logger = logger
|
||||
self.logger.remove()
|
||||
self.logger.add(
|
||||
sys.stdout,
|
||||
level="DEBUG",
|
||||
format="<green>{time:YYYYMMDD HH:mm:ss}</green> | "
|
||||
"{process.name} | "
|
||||
"{thread.name} | "
|
||||
"<cyan>{module}</cyan>.<cyan>{function}</cyan>"
|
||||
":<cyan>{line}</cyan> | "
|
||||
"<level>{level}</level>: "
|
||||
"<level>{message}</level>",
|
||||
)
|
||||
|
||||
def get_logger(self):
|
||||
return self.logger
|
||||
|
||||
|
||||
log = MyLogger().get_logger()
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("str.pdf"["str.pdf".rindex(".") :])
|
||||
|
||||
def test():
|
||||
try:
|
||||
print(3)
|
||||
except ZeroDivisionError as e:
|
||||
log.exception(e)
|
||||
|
||||
test()
|
||||
@@ -0,0 +1 @@
|
||||
"""Web 前端静态资源与本地 API 服务（见 ``server`` 模块）。"""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
Binary file not shown.
|
After Width: | Height: | Size: 137 KiB |
@@ -0,0 +1 @@
|
||||
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1776231079896" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="1934" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><path d="M507.904 391.168q-72.704 0-129.024-16.896t-86.016-33.28q-1.024-1.024-1.024-5.12l0-8.192q0-2.048 0.512-0.512t0.512 13.824l0 523.264-115.712 0 0-614.4q0-20.48 7.168-43.52t34.816-34.304q14.336-6.144 28.16-4.608t24.576 6.144 18.432 10.24 10.752 7.68q19.456 15.36 39.936 28.672t44.544 24.064 53.76 16.896 67.584 6.144l4.096 0 1.024 0 4.096 0q37.888 0 67.584-6.144t54.272-16.896 45.056-24.064 38.912-28.672q3.072-2.048 10.752-7.68t18.944-10.24 25.088-6.144 28.16 4.608q26.624 11.264 34.304 34.304t7.68 43.52l0 614.4-116.736 0 0-523.264q0-12.288 1.024-13.312 1.024 3.072 0.512 7.68t-1.536 5.632q-14.336 8.192-36.864 16.896t-50.688 16.384-60.928 12.288-66.56 4.608l-7.168 0z" p-id="1935" fill="#1296db"></path></svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
@@ -0,0 +1,4 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none">
|
||||
<path d="M12 3.25a3.75 3.75 0 0 1 3.75 3.75v5a3.75 3.75 0 1 1-7.5 0V7A3.75 3.75 0 0 1 12 3.25Z" fill="#0e7490"/>
|
||||
<path d="M6.5 10.75a.75.75 0 0 1 .75.75 4.75 4.75 0 0 0 9.5 0 .75.75 0 0 1 1.5 0 6.25 6.25 0 0 1-5.5 6.203V20h2a.75.75 0 0 1 0 1.5h-5.5a.75.75 0 0 1 0-1.5h2v-2.297A6.25 6.25 0 0 1 5.75 11.5a.75.75 0 0 1 .75-.75Z" fill="#0f172a" opacity=".86"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 441 B |
@@ -0,0 +1,132 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover" />
|
||||
<title>minor · Chatbot</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=ZCOOL+KuaiLe&display=swap"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
<link rel="icon" href="/image/favicon.svg" sizes="any" />
|
||||
<link rel="stylesheet" href="/app.css" />
|
||||
</head>
|
||||
<body>
|
||||
<div class="bg-scene" id="bgScene" aria-hidden="true">
|
||||
<div class="bg-image" id="bgImageLayer"></div>
|
||||
<div class="bg-vignette" aria-hidden="true"></div>
|
||||
</div>
|
||||
<div class="app-root" id="app">
|
||||
<div class="drawer-backdrop" id="drawerBackdrop" aria-hidden="true"></div>
|
||||
<aside class="expand-rail" id="expandRail" aria-hidden="true">
|
||||
<button type="button" class="rail-btn" id="expandBtn" title="展开侧栏">⟩</button>
|
||||
</aside>
|
||||
<aside class="sidebar" id="sidebar">
|
||||
<div class="sidebar-head">
|
||||
<h2 class="sidebar-title">会话</h2>
|
||||
<button type="button" class="btn btn-primary" id="newSessionBtn">＋ 新建会话</button>
|
||||
<button type="button" class="btn btn-danger" id="deleteSessionBtn">删除会话</button>
|
||||
</div>
|
||||
<div class="sidebar-list-wrap">
|
||||
<ul class="session-list" id="sessionList" role="listbox" aria-label="会话列表"></ul>
|
||||
</div>
|
||||
<p class="sidebar-hint">点击切换会话；新建为秒级时间 ID。</p>
|
||||
<div class="sidebar-bg-panel" aria-label="背景与景深">
|
||||
<p class="sidebar-bg-title">页面背景</p>
|
||||
<label class="bg-control">
|
||||
<span class="bg-control-label">景深模糊</span>
|
||||
<input type="range" id="dofRange" min="0" max="24" value="0" step="0.5" />
|
||||
<span class="bg-control-value" id="dofValue">0 px</span>
|
||||
</label>
|
||||
<label class="bg-control">
|
||||
<span class="bg-control-label">暗角</span>
|
||||
<input type="range" id="vignetteRange" min="0" max="100" value="35" step="1" />
|
||||
<span class="bg-control-value" id="vignetteValue">35 %</span>
|
||||
</label>
|
||||
<p class="sidebar-bg-tip">替换 <code>web/image/background.jpg</code> 可换图；设置会保存在本机浏览器。</p>
|
||||
</div>
|
||||
<div class="sidebar-collapse">
|
||||
<button type="button" class="collapse-fab" id="collapseBtn" title="收起侧栏">⟨</button>
|
||||
</div>
|
||||
</aside>
|
||||
<main class="main" id="mainPanel">
|
||||
<header class="main-header">
|
||||
<button
|
||||
type="button"
|
||||
class="menu-btn"
|
||||
id="menuBtn"
|
||||
aria-expanded="false"
|
||||
aria-controls="sidebar"
|
||||
title="会话与设置"
|
||||
>
|
||||
☰
|
||||
</button>
|
||||
<div class="main-header-text" id="mainHeaderText">
|
||||
<h1 class="title">minor · Chatbot</h1>
|
||||
<p class="subtitle">本地多模态对话（Web）</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
class="header-toggle"
|
||||
id="headerToggleBtn"
|
||||
aria-expanded="true"
|
||||
aria-controls="mainHeaderText"
|
||||
title="收起标题栏"
|
||||
>
|
||||
▴
|
||||
</button>
|
||||
</header>
|
||||
<section class="chat-shell">
|
||||
<div class="chat-placeholder" id="chatPlaceholder">
|
||||
<h3>开始对话</h3>
|
||||
<p>输入文字，或上传<strong>图片</strong> / <strong>音频</strong> 后点击发送。</p>
|
||||
<p class="muted">支持用户音频输入、助手文本/语音/双模输出、会话本地媒体持久化。</p>
|
||||
</div>
|
||||
<div class="chat-messages" id="chatMessages" hidden></div>
|
||||
</section>
|
||||
<footer class="composer">
|
||||
<div class="composer-attachments" id="attachmentPreview"></div>
|
||||
<div class="composer-row">
|
||||
<div class="mm-input" role="group" aria-label="多模态输入">
|
||||
<div class="composer-input-shell">
|
||||
<textarea
|
||||
id="textInput"
|
||||
rows="2"
|
||||
placeholder="输入消息或添加附件，Enter 发送（Shift+Enter 换行）"
|
||||
></textarea>
|
||||
<button type="button" class="mic-btn" id="micBtn" title="使用麦克风录音" aria-pressed="false" aria-label="使用麦克风录音">
|
||||
<img src="/image/microphone.svg" alt="" aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
<div class="mm-actions">
|
||||
<label class="btn btn-ghost file-btn">
|
||||
图片
|
||||
<input type="file" id="imageInput" accept="image/png,image/jpeg,image/jpg,image/gif,image/webp,image/bmp,.png,.jpg,.jpeg,.gif,.webp,.bmp" multiple hidden />
|
||||
</label>
|
||||
<label class="btn btn-ghost file-btn">
|
||||
音频
|
||||
<input type="file" id="audioInput" accept="audio/*,.wav,.mp3,.m4a,.flac,.ogg" hidden />
|
||||
</label>
|
||||
<label class="output-mode-label" for="outputModeSelect">输出模式</label>
|
||||
<select id="outputModeSelect" class="output-mode-select" aria-label="Agent 输出模式">
|
||||
<option value="text">纯文字</option>
|
||||
<option value="audio">纯语音</option>
|
||||
<option value="both">both</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="composer-send">
|
||||
<button type="button" class="btn btn-primary" id="sendBtn">发送</button>
|
||||
<button type="button" class="btn btn-secondary" id="clearBtn">清空</button>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
</main>
|
||||
</div>
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/dompurify@3.1.6/dist/purify.min.js"></script>
|
||||
<script src="/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,406 @@
|
||||
"""
|
||||
为 ``web`` 目录下的静态前端提供 API：会话列表、多模态发送（与 ``chatbot_agent`` 一致）。
|
||||
|
||||
从仓库 ``Agent`` 目录、且 ``PYTHONPATH`` 包含 ``src`` 时启动::
|
||||
|
||||
uvicorn agent.web.server:app --reload --host 127.0.0.1 --port 8765
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse
|
||||
_SRC = Path(__file__).resolve().parent.parent.parent
|
||||
if str(_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
from agent import chatbot_agent as ca # noqa: E402
|
||||
from agent.web import ui_session as ui # noqa: E402
|
||||
from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS # noqa: E402
|
||||
|
||||
_WEB_DIR = Path(__file__).resolve().parent
|
||||
_IMAGE_DIR = _WEB_DIR / "image"
|
||||
|
||||
_AUDIO_EXT = frozenset({".wav", ".mp3", ".m4a", ".flac", ".ogg", ".webm"})
|
||||
|
||||
|
||||
def _data_url_for_file(fp: str) -> str | None:
|
||||
"""将本地文件读入为 data URL，供前端直接展示（不单独提供文件下载路由）。"""
|
||||
if not fp or not os.path.isfile(fp):
|
||||
return None
|
||||
mime, _ = mimetypes.guess_type(fp)
|
||||
if not mime:
|
||||
ext = os.path.splitext(fp)[1].lower()
|
||||
if ext in IMAGE_FILE_EXTENSIONS:
|
||||
mime = {".png": "image/png", ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp"}.get(
|
||||
ext, "image/jpeg"
|
||||
)
|
||||
elif ext in _AUDIO_EXT:
|
||||
mime = {
|
||||
".wav": "audio/wav",
|
||||
".mp3": "audio/mpeg",
|
||||
".m4a": "audio/mp4",
|
||||
".flac": "audio/flac",
|
||||
".ogg": "audio/ogg",
|
||||
}.get(ext, "application/octet-stream")
|
||||
else:
|
||||
mime = "application/octet-stream"
|
||||
try:
|
||||
with open(fp, "rb") as f:
|
||||
raw = f.read()
|
||||
b64 = base64.standard_b64encode(raw).decode("ascii")
|
||||
except OSError:
|
||||
return None
|
||||
return f"data:{mime};base64,{b64}"
|
||||
|
||||
|
||||
def _serialize_content(content: Any, *, meta: dict[str, Any] | None = None, role: str | None = None) -> Any:
|
||||
"""将一条消息中的 content 转为 JSON 友好结构；文件以内联 data URL 输出。"""
|
||||
meta = meta or {}
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if not isinstance(content, list):
|
||||
return str(content)
|
||||
parts: list[dict[str, Any]] = []
|
||||
for piece in content:
|
||||
if isinstance(piece, str) and piece.strip():
|
||||
parts.append({"type": "text", "text": piece.strip()})
|
||||
continue
|
||||
if not isinstance(piece, dict):
|
||||
continue
|
||||
if piece.get("type") == "text":
|
||||
parts.append({"type": "text", "text": piece.get("text") or ""})
|
||||
continue
|
||||
fp = piece.get("path") or (piece.get("file") or {}).get("path")
|
||||
if piece.get("type") == "image_url":
|
||||
fp = fp or (piece.get("image_url") or {}).get("url")
|
||||
if piece.get("type") == "audio":
|
||||
fp = fp or piece.get("audio_path")
|
||||
if not fp or not os.path.isfile(fp):
|
||||
continue
|
||||
data_url = _data_url_for_file(fp)
|
||||
if not data_url:
|
||||
continue
|
||||
ext = os.path.splitext(fp)[1].lower()
|
||||
if ext in IMAGE_FILE_EXTENSIONS:
|
||||
parts.append({"type": "image", "url": data_url, "path": fp, "name": os.path.basename(fp)})
|
||||
elif ext in _AUDIO_EXT:
|
||||
audio_meta = meta.get("audio") if isinstance(meta.get("audio"), dict) else {}
|
||||
parts.append({
|
||||
"type": "audio",
|
||||
"url": data_url,
|
||||
"path": fp,
|
||||
"name": os.path.basename(fp),
|
||||
"autoplay": bool(meta.get("autoplay") or audio_meta.get("autoplay")) if role == "assistant" else False,
|
||||
"duration_seconds": audio_meta.get("duration_seconds"),
|
||||
})
|
||||
else:
|
||||
parts.append({"type": "file", "url": data_url, "path": fp, "name": os.path.basename(fp)})
|
||||
return parts
|
||||
|
||||
|
||||
def _serialize_messages(hist: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
|
||||
hist = hist or []
|
||||
# 找到最后一条 assistant 消息的索引，用于控制自动播放
|
||||
last_assistant_idx = -1
|
||||
for i in range(len(hist) - 1, -1, -1):
|
||||
if hist[i].get("role") == "assistant":
|
||||
last_assistant_idx = i
|
||||
break
|
||||
|
||||
out = []
|
||||
for idx, m in enumerate(hist):
|
||||
role = m.get("role")
|
||||
content = m.get("content")
|
||||
meta = dict(m.get("meta") or {})
|
||||
meta.setdefault("output_type", "text" if role != "assistant" else meta.get("output_type") or "text")
|
||||
if role == "assistant":
|
||||
meta.setdefault("autoplay", bool((meta.get("audio") or {}).get("autoplay") or meta.get("autoplay")))
|
||||
|
||||
serialized_content = _serialize_content(content, meta=meta, role=role)
|
||||
|
||||
# ---------- 音频部件处理 ----------
|
||||
if role == "assistant" and meta.get("audio"):
|
||||
audio_meta = meta.get("audio") if isinstance(meta.get("audio"), dict) else {}
|
||||
# 检查 content 中是否已经包含了音频类型的部件（例如从 manifest 加载时生成的）
|
||||
already_has_audio = False
|
||||
if isinstance(serialized_content, list):
|
||||
for part in serialized_content:
|
||||
if isinstance(part, dict) and part.get("type") == "audio":
|
||||
already_has_audio = True
|
||||
# 修正历史消息的 autoplay 为 False（仅最新一条可允许）
|
||||
if idx != last_assistant_idx:
|
||||
part["autoplay"] = False
|
||||
break
|
||||
|
||||
# 只有 content 中还没有音频部件时，才追加额外的 audio_part
|
||||
if not already_has_audio:
|
||||
is_latest = (idx == last_assistant_idx)
|
||||
autoplay_final = is_latest and (meta.get("autoplay") or audio_meta.get("autoplay"))
|
||||
audio_part = {
|
||||
"type": "audio",
|
||||
"url": _data_url_for_file(audio_meta.get("path")),
|
||||
"name": "语音回复",
|
||||
"autoplay": autoplay_final,
|
||||
"duration_seconds": audio_meta.get("duration_seconds"),
|
||||
}
|
||||
if audio_part["url"]:
|
||||
if isinstance(serialized_content, list):
|
||||
serialized_content.append(audio_part)
|
||||
elif serialized_content:
|
||||
serialized_content = [{"type": "text", "text": serialized_content}, audio_part]
|
||||
else:
|
||||
serialized_content = [audio_part]
|
||||
|
||||
out.append({"role": role, "content": serialized_content, "meta": meta})
|
||||
return out
|
||||
|
||||
|
||||
def _bootstrap() -> tuple[str, list[str], list[dict[str, Any]]]:
|
||||
ids = ui.list_session_ids_ordered()
|
||||
if ids:
|
||||
sid = ids[0]
|
||||
hist = ui.load_ui_messages_for_session(sid)
|
||||
else:
|
||||
sid = ui.new_timestamp_session_id()
|
||||
hist = []
|
||||
ids = [sid]
|
||||
return sid, ids, hist
|
||||
|
||||
|
||||
app = FastAPI(title="minor web chat", version="1.0.0")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/bootstrap")
|
||||
def api_bootstrap() -> dict[str, Any]:
|
||||
sid, ids, hist = _bootstrap()
|
||||
return {"sessionId": sid, "sessions": ids, "messages": _serialize_messages(hist)}
|
||||
|
||||
|
||||
@app.post("/api/sessions/new")
|
||||
def api_new_session() -> dict[str, Any]:
|
||||
sid = ui.new_timestamp_session_id()
|
||||
ids = ui.list_session_ids_ordered()
|
||||
ids = [sid] + [x for x in ids if x != sid]
|
||||
return {"sessionId": sid, "sessions": ids, "messages": []}
|
||||
|
||||
|
||||
@app.get("/api/sessions/{session_id}/messages")
|
||||
def api_session_messages(session_id: str) -> dict[str, Any]:
|
||||
if not session_id:
|
||||
raise HTTPException(status_code=400, detail="缺少 session_id")
|
||||
hist = ui.load_ui_messages_for_session(session_id)
|
||||
ids = ui.list_session_ids_ordered()
|
||||
if session_id not in ids:
|
||||
ids = [session_id, *ids]
|
||||
return {"sessionId": session_id, "sessions": ids, "messages": _serialize_messages(hist)}
|
||||
|
||||
|
||||
@app.post("/api/sessions/clear")
|
||||
def api_clear_session(session_id: str = Form(...)) -> dict[str, Any]:
|
||||
if not session_id:
|
||||
raise HTTPException(status_code=400, detail="缺少 session_id")
|
||||
messages, _ = ui.clear_ui_and_manifest(session_id)
|
||||
ids = ui.list_session_ids_ordered()
|
||||
if session_id not in ids:
|
||||
ids = [session_id, *ids]
|
||||
return {"sessionId": session_id, "sessions": ids, "messages": _serialize_messages(messages)}
|
||||
|
||||
|
||||
@app.post("/api/sessions/delete")
|
||||
def api_delete_session(session_id: str = Form(...)) -> dict[str, Any]:
|
||||
if not session_id:
|
||||
raise HTTPException(status_code=400, detail="缺少 session_id")
|
||||
ui.delete_session(session_id)
|
||||
sid, ids, hist = _bootstrap()
|
||||
return {"sessionId": sid, "sessions": ids, "messages": _serialize_messages(hist)}
|
||||
|
||||
|
||||
def _materialize_uploads_to_temp(
|
||||
images: list[UploadFile] | None,
|
||||
audio: UploadFile | None,
|
||||
text: str,
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
"""将上传文件落盘到临时目录，返回 ``(query_dict, tmp_root)``；调用方须在 finally 中删除 ``tmp_root``。"""
|
||||
tmp_root = tempfile.mkdtemp(prefix="web_mm_")
|
||||
file_paths: list[str] = []
|
||||
for uf in images or []:
|
||||
if uf is None or not uf.filename:
|
||||
continue
|
||||
ext = Path(uf.filename).suffix or ""
|
||||
dest = os.path.join(tmp_root, f"{uuid.uuid4().hex}{ext}")
|
||||
with open(dest, "wb") as f:
|
||||
shutil.copyfileobj(uf.file, f)
|
||||
file_paths.append(dest)
|
||||
|
||||
if audio is not None and audio.filename:
|
||||
ext = Path(audio.filename).suffix or ".bin"
|
||||
dest = os.path.join(tmp_root, f"{uuid.uuid4().hex}{ext}")
|
||||
with open(dest, "wb") as f:
|
||||
shutil.copyfileobj(audio.file, f)
|
||||
file_paths.append(dest)
|
||||
|
||||
query = {"text": (text or "").strip(), "files": file_paths}
|
||||
return query, tmp_root
|
||||
|
||||
|
||||
def _chat_start_core(session_id: str, query: dict[str, Any]) -> dict[str, Any]:
|
||||
"""追加 user、复制附件到会话目录并写 manifest（不调用模型）。"""
|
||||
hist = ui.load_ui_messages_for_session(session_id)
|
||||
before = len(hist or [])
|
||||
new_hist, cleared = ui.add_message(hist or [], query)
|
||||
if cleared is None:
|
||||
raise HTTPException(status_code=400, detail="请输入文字或上传图片/音频")
|
||||
if len(new_hist) == before:
|
||||
raise HTTPException(status_code=400, detail="请输入文字或上传图片/音频")
|
||||
|
||||
ui.rewrite_user_files_for_last_turn(new_hist, session_id)
|
||||
ui.sync_manifest_from_chat_history(session_id, new_hist, "")
|
||||
|
||||
ids = ui.list_session_ids_ordered()
|
||||
if session_id not in ids:
|
||||
ids = [session_id, *ids]
|
||||
return {
|
||||
"sessionId": session_id,
|
||||
"sessions": ids,
|
||||
"messages": _serialize_messages(new_hist),
|
||||
}
|
||||
|
||||
|
||||
def _chat_complete_core(session_id: str, output_type: str = "text") -> dict[str, Any]:
|
||||
"""对 manifest 中末条 user 执行视觉解析与 ``chain_with_history.invoke``。"""
|
||||
hist = ui.load_ui_messages_for_session(session_id)
|
||||
if not hist or hist[-1].get("role") != "user":
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="没有待完成的用户消息，请先调用 /api/chat/start",
|
||||
)
|
||||
new_hist = ca.execute_agent([*hist], session_id, output_type=output_type)
|
||||
ui.prune_memory_session_ids()
|
||||
ids = ui.list_session_ids_ordered()
|
||||
if session_id not in ids:
|
||||
ids = [session_id, *ids]
|
||||
return {
|
||||
"sessionId": session_id,
|
||||
"sessions": ids,
|
||||
"messages": _serialize_messages(new_hist),
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/chat/start")
|
||||
async def api_chat_start(
|
||||
session_id: str = Form(...),
|
||||
text: str = Form(""),
|
||||
images: list[UploadFile] | None = File(None),
|
||||
audio: UploadFile | None = File(None),
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
仅追加用户消息并持久化到 manifest（含附件落盘）；不调用模型。
|
||||
前端应先调用本接口再调用 ``/api/chat/complete``。
|
||||
"""
|
||||
if not session_id:
|
||||
raise HTTPException(status_code=400, detail="缺少 session_id")
|
||||
|
||||
query, tmp_root = _materialize_uploads_to_temp(images, audio, text)
|
||||
try:
|
||||
return _chat_start_core(session_id, query)
|
||||
finally:
|
||||
shutil.rmtree(tmp_root, ignore_errors=True)
|
||||
|
||||
|
||||
@app.post("/api/chat/complete")
|
||||
async def api_chat_complete(
|
||||
session_id: str = Form(...),
|
||||
output_type: str = Form("text"),
|
||||
) -> dict[str, Any]:
|
||||
"""对当前会话最近一次 ``/api/chat/start`` 提交执行模型推理并返回完整消息列表。"""
|
||||
if not session_id:
|
||||
raise HTTPException(status_code=400, detail="缺少 session_id")
|
||||
return _chat_complete_core(session_id, output_type=output_type)
|
||||
|
||||
|
||||
@app.post("/api/chat")
|
||||
async def api_chat(
|
||||
session_id: str = Form(...),
|
||||
text: str = Form(""),
|
||||
images: list[UploadFile] | None = File(None),
|
||||
audio: UploadFile | None = File(None),
|
||||
output_type: str = Form("text"),
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
兼容旧客户端：等价于依次 ``/api/chat/start`` + ``/api/chat/complete``（仍阻塞至推理结束）。
|
||||
"""
|
||||
if not session_id:
|
||||
raise HTTPException(status_code=400, detail="缺少 session_id")
|
||||
|
||||
query, tmp_root = _materialize_uploads_to_temp(images, audio, text)
|
||||
try:
|
||||
_chat_start_core(session_id, query)
|
||||
return _chat_complete_core(session_id, output_type=output_type)
|
||||
finally:
|
||||
shutil.rmtree(tmp_root, ignore_errors=True)
|
||||
|
||||
|
||||
|
||||
# 先注册 API，再挂静态文件，避免 ``/`` 抢路由
|
||||
@app.get("/api/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/favicon.ico")
|
||||
def favicon() -> FileResponse:
|
||||
for name, mime in (
|
||||
("favicon.ico", "image/x-icon"),
|
||||
("favicon.png", "image/png"),
|
||||
("favicon.svg", "image/svg+xml"),
|
||||
):
|
||||
path = _IMAGE_DIR / name
|
||||
if path.is_file():
|
||||
return FileResponse(path, media_type=mime)
|
||||
raise HTTPException(status_code=404, detail="web/image 目录下缺少 favicon.ico / favicon.png / favicon.svg")
|
||||
|
||||
|
||||
@app.get("/image/{asset_name}")
|
||||
def serve_image_asset(asset_name: str) -> FileResponse:
|
||||
path = (_IMAGE_DIR / asset_name).resolve()
|
||||
try:
|
||||
path.relative_to(_IMAGE_DIR.resolve())
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=404, detail="图片资源不存在") from exc
|
||||
if not path.is_file():
|
||||
raise HTTPException(status_code=404, detail=f"web/image 目录下缺少 {asset_name}")
|
||||
mime, _ = mimetypes.guess_type(str(path))
|
||||
return FileResponse(path, media_type=mime or "application/octet-stream")
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def index_page() -> FileResponse:
|
||||
return FileResponse(_WEB_DIR / "index.html")
|
||||
|
||||
|
||||
@app.get("/app.css")
|
||||
def serve_css() -> FileResponse:
|
||||
return FileResponse(_WEB_DIR / "app.css")
|
||||
|
||||
|
||||
@app.get("/app.js")
|
||||
def serve_js() -> FileResponse:
|
||||
return FileResponse(_WEB_DIR / "app.js")
|
||||
@@ -0,0 +1,461 @@
|
||||
"""
|
||||
Web/API 与会话 UI 状态：会话 id 列表、manifest、多模态 user 消息结构、附件落盘。
|
||||
与 LangChain 链解耦；execute_agent 仍在 chatbot_agent 中，此处提供其依赖的同步与解析函数。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from agent.utils.agent_utils import (
|
||||
CHAT_HISTORY_DB_FS,
|
||||
MULTI_INPUTS_ROOT,
|
||||
assistant_text,
|
||||
extract_result_from_invoke,
|
||||
)
|
||||
from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS
|
||||
from agent.chatbot_agent import get_history
|
||||
|
||||
_AUDIO_EXTENSIONS = {".wav", ".mp3", ".m4a", ".flac", ".ogg"}
|
||||
_memory_session_ids: set[str] = set()
|
||||
|
||||
|
||||
def manifest_path(session_id: str) -> str:
|
||||
return os.path.join(MULTI_INPUTS_ROOT, session_id, "ui_messages.json")
|
||||
|
||||
|
||||
def session_dir(session_id: str) -> str:
|
||||
return os.path.join(MULTI_INPUTS_ROOT, session_id)
|
||||
|
||||
|
||||
def list_session_ids_ordered() -> list[str]:
|
||||
if not os.path.isfile(CHAT_HISTORY_DB_FS):
|
||||
return []
|
||||
conn = sqlite3.connect(CHAT_HISTORY_DB_FS)
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"SELECT session_id FROM message_store GROUP BY session_id ORDER BY MAX(id) DESC"
|
||||
)
|
||||
return [row[0] for row in cur.fetchall() if row[0]]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def new_timestamp_session_id() -> str:
|
||||
base = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
existing = set(list_session_ids_ordered()) | _memory_session_ids
|
||||
if base not in existing:
|
||||
_memory_session_ids.add(base)
|
||||
return base
|
||||
n = 1
|
||||
while True:
|
||||
cand = f"{base}_{n}"
|
||||
if cand not in existing:
|
||||
_memory_session_ids.add(cand)
|
||||
return cand
|
||||
n += 1
|
||||
|
||||
|
||||
def prune_memory_session_ids() -> None:
|
||||
persisted = set(list_session_ids_ordered())
|
||||
for sid in list(_memory_session_ids):
|
||||
if sid in persisted:
|
||||
_memory_session_ids.discard(sid)
|
||||
|
||||
|
||||
def delete_session(session_id: str) -> None:
|
||||
if not session_id:
|
||||
return
|
||||
if os.path.isfile(CHAT_HISTORY_DB_FS):
|
||||
conn = sqlite3.connect(CHAT_HISTORY_DB_FS)
|
||||
try:
|
||||
conn.execute("DELETE FROM message_store WHERE session_id = ?", (session_id,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
_memory_session_ids.discard(session_id)
|
||||
shutil.rmtree(session_dir(session_id), ignore_errors=True)
|
||||
|
||||
|
||||
def _lc_content_for_ui_messages(content) -> str | list:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
t = assistant_text(content)
|
||||
return t if t else str(content)
|
||||
return str(content)
|
||||
|
||||
|
||||
def _load_ui_messages_from_db(session_id: str) -> list:
|
||||
if not session_id:
|
||||
return []
|
||||
rows: list[dict] = []
|
||||
for m in get_history(session_id).messages:
|
||||
if m.type == "human":
|
||||
rows.append({"role": "user", "content": _lc_content_for_ui_messages(m.content)})
|
||||
elif m.type in ("ai", "assistant"):
|
||||
text = assistant_text(m.content) or extract_result_from_invoke(m)
|
||||
rows.append({"role": "assistant", "content": text or "（空回复）"})
|
||||
return rows
|
||||
|
||||
|
||||
def _build_user_chat_content(query) -> str | list | None:
|
||||
parts: list = []
|
||||
for filepath in query.get("files", []) or []:
|
||||
if filepath:
|
||||
parts.append({"path": filepath})
|
||||
text = (query.get("text") or "").strip()
|
||||
if text:
|
||||
parts.append({"type": "text", "text": text})
|
||||
if not parts:
|
||||
return None
|
||||
if len(parts) == 1 and isinstance(parts[0], dict) and parts[0].get("type") == "text":
|
||||
return parts[0]["text"]
|
||||
return parts
|
||||
|
||||
|
||||
def collect_media_paths_and_query_text(user_msg: dict) -> tuple[list[str], list[str], str]:
|
||||
raw = user_msg.get("content")
|
||||
text_segments: list[str] = []
|
||||
image_paths: list[str] = []
|
||||
audio_paths: list[str] = []
|
||||
|
||||
if isinstance(raw, str):
|
||||
t = raw.strip()
|
||||
if t:
|
||||
text_segments.append(t)
|
||||
return image_paths, audio_paths, "\n".join(text_segments).strip()
|
||||
|
||||
if not isinstance(raw, list):
|
||||
return image_paths, audio_paths, ""
|
||||
|
||||
for piece in raw:
|
||||
if isinstance(piece, str):
|
||||
t = piece.strip()
|
||||
if t:
|
||||
text_segments.append(t)
|
||||
continue
|
||||
if not isinstance(piece, dict):
|
||||
continue
|
||||
if piece.get("type") == "text":
|
||||
text_content = (piece.get("text") or "").strip()
|
||||
if text_content:
|
||||
text_segments.append(text_content)
|
||||
continue
|
||||
file_path = piece.get("path") or (piece.get("file") or {}).get("path")
|
||||
if piece.get("type") == "image_url":
|
||||
file_path = file_path or (piece.get("image_url") or {}).get("url")
|
||||
if piece.get("type") == "audio":
|
||||
file_path = file_path or piece.get("audio_path")
|
||||
if not file_path:
|
||||
continue
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
if ext in IMAGE_FILE_EXTENSIONS:
|
||||
image_paths.append(file_path)
|
||||
elif ext in _AUDIO_EXTENSIONS:
|
||||
audio_paths.append(file_path)
|
||||
else:
|
||||
text_segments.append(f"[用户上传了附件文件: {os.path.basename(file_path)}]")
|
||||
|
||||
return image_paths, audio_paths, "\n".join(text_segments).strip()
|
||||
|
||||
|
||||
def rewrite_user_message_files_to_session_dir(user_msg: dict, session_id: str) -> None:
|
||||
dest_dir = session_dir(session_id)
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
raw = user_msg.get("content")
|
||||
if not isinstance(raw, list):
|
||||
return
|
||||
new_list: list = []
|
||||
for piece in raw:
|
||||
if isinstance(piece, str) or not isinstance(piece, dict):
|
||||
new_list.append(piece)
|
||||
continue
|
||||
if piece.get("type") == "text":
|
||||
new_list.append(piece)
|
||||
continue
|
||||
src = piece.get("path") or (piece.get("file") or {}).get("path")
|
||||
if piece.get("type") == "image_url":
|
||||
src = src or (piece.get("image_url") or {}).get("url")
|
||||
if piece.get("type") == "audio":
|
||||
src = src or piece.get("audio_path")
|
||||
if not src or not os.path.isfile(src):
|
||||
new_list.append(piece)
|
||||
continue
|
||||
dest_dir_norm = os.path.normpath(dest_dir) + os.sep
|
||||
src_norm = os.path.normpath(src)
|
||||
if src_norm.startswith(dest_dir_norm):
|
||||
new_list.append(piece)
|
||||
continue
|
||||
ext = os.path.splitext(src)[1] or ""
|
||||
dest = os.path.join(dest_dir, f"{uuid.uuid4().hex}{ext}")
|
||||
shutil.copy2(src, dest)
|
||||
new_piece = dict(piece)
|
||||
new_piece["path"] = dest
|
||||
if piece.get("type") == "image_url":
|
||||
new_piece["image_url"] = {"url": dest}
|
||||
if piece.get("type") == "audio":
|
||||
new_piece["audio_path"] = dest
|
||||
new_list.append(new_piece)
|
||||
user_msg["content"] = new_list
|
||||
|
||||
|
||||
def _serialize_user_msg_for_manifest(user_msg: dict) -> dict:
|
||||
raw = user_msg.get("content")
|
||||
content_list: list[dict] = []
|
||||
if isinstance(raw, str):
|
||||
if raw.strip():
|
||||
content_list.append({"type": "text", "text": raw})
|
||||
elif isinstance(raw, list):
|
||||
for piece in raw:
|
||||
if isinstance(piece, str) and piece.strip():
|
||||
content_list.append({"type": "text", "text": piece.strip()})
|
||||
elif isinstance(piece, dict):
|
||||
if piece.get("type") == "text":
|
||||
content_list.append({"type": "text", "text": piece.get("text") or ""})
|
||||
continue
|
||||
fp = piece.get("path") or (piece.get("file") or {}).get("path")
|
||||
if piece.get("type") == "image_url":
|
||||
fp = fp or (piece.get("image_url") or {}).get("url")
|
||||
if piece.get("type") == "audio":
|
||||
fp = fp or piece.get("audio_path")
|
||||
if not fp:
|
||||
continue
|
||||
ext = os.path.splitext(fp)[1].lower()
|
||||
try:
|
||||
rel = os.path.relpath(os.path.normpath(fp), os.path.normpath(MULTI_INPUTS_ROOT))
|
||||
except ValueError:
|
||||
rel = fp.replace("\\", "/")
|
||||
if ext in IMAGE_FILE_EXTENSIONS:
|
||||
part_type = "image_url"
|
||||
elif ext in _AUDIO_EXTENSIONS:
|
||||
part_type = "audio"
|
||||
else:
|
||||
part_type = "file"
|
||||
content_list.append({"type": part_type, "relpath": rel.replace("\\", "/")})
|
||||
# 不再写入 image_description 字段
|
||||
return {"content": content_list}
|
||||
|
||||
|
||||
def _manifest_user_entry_to_content(user_entry: dict) -> str | list:
|
||||
root_norm = os.path.normpath(MULTI_INPUTS_ROOT)
|
||||
parts_out: list = []
|
||||
any_image_missing = False
|
||||
had_image_parts = False
|
||||
|
||||
# 新格式直接取 "content"
|
||||
content_data = user_entry.get("content")
|
||||
if content_data is None:
|
||||
return ""
|
||||
|
||||
for p in content_data:
|
||||
part_type = p.get("type")
|
||||
if part_type == "text":
|
||||
t = (p.get("text") or "").strip()
|
||||
if t:
|
||||
parts_out.append({"type": "text", "text": t})
|
||||
continue
|
||||
rel = (p.get("relpath") or "").replace("/", os.sep)
|
||||
abs_p = os.path.normpath(os.path.join(root_norm, rel))
|
||||
try:
|
||||
under = os.path.commonpath([abs_p, root_norm]) == root_norm
|
||||
except ValueError:
|
||||
under = False
|
||||
if not under:
|
||||
if part_type == "image_url":
|
||||
any_image_missing = True
|
||||
had_image_parts = True
|
||||
continue
|
||||
if part_type == "image_url":
|
||||
had_image_parts = True
|
||||
if os.path.isfile(abs_p):
|
||||
parts_out.append({"type": "image_url", "image_url": {"url": abs_p}, "path": abs_p})
|
||||
else:
|
||||
any_image_missing = True
|
||||
elif part_type == "audio":
|
||||
if os.path.isfile(abs_p):
|
||||
parts_out.append({"type": "audio", "audio_path": abs_p, "path": abs_p})
|
||||
else:
|
||||
parts_out.append({"type": "text", "text": f"[语音已丢失: {os.path.basename(rel) or rel}]"})
|
||||
elif part_type == "file":
|
||||
if os.path.isfile(abs_p):
|
||||
parts_out.append({"path": abs_p})
|
||||
else:
|
||||
parts_out.append({"type": "text", "text": f"[附件已丢失: {os.path.basename(rel) or rel}]"})
|
||||
|
||||
# 不再处理 image_description（新格式中没有）
|
||||
if had_image_parts and any_image_missing:
|
||||
parts_out.append({"type": "text", "text": "[图片已丢失]"})
|
||||
|
||||
if not parts_out:
|
||||
return ""
|
||||
if len(parts_out) == 1 and parts_out[0].get("type") == "text":
|
||||
return parts_out[0]["text"]
|
||||
return parts_out
|
||||
|
||||
|
||||
def _serialize_assistant_entry(assistant_msg: dict | None) -> dict:
|
||||
ac = (assistant_msg or {}).get("content", "")
|
||||
meta = dict((assistant_msg or {}).get("meta") or {})
|
||||
if isinstance(ac, str):
|
||||
atext = ac
|
||||
else:
|
||||
atext = assistant_text(ac) or extract_result_from_invoke(assistant_msg) or "（空回复）"
|
||||
audio_relpath = None
|
||||
audio = meta.get("audio")
|
||||
if isinstance(audio, dict) and audio.get("path"):
|
||||
try:
|
||||
audio_relpath = os.path.relpath(
|
||||
os.path.normpath(audio["path"]), os.path.normpath(MULTI_INPUTS_ROOT)
|
||||
)
|
||||
except ValueError:
|
||||
audio_relpath = str(audio["path"]).replace("\\", "/")
|
||||
return {
|
||||
"text": atext,
|
||||
"output_type": meta.get("output_type") or "text",
|
||||
"autoplay": bool(meta.get("autoplay") or (audio or {}).get("autoplay")),
|
||||
"audio_duration_seconds": (audio or {}).get("duration_seconds") if isinstance(audio, dict) else None,
|
||||
"audio_relpath": audio_relpath.replace("\\", "/") if audio_relpath else None,
|
||||
}
|
||||
|
||||
|
||||
def sync_manifest_from_chat_history(session_id: str, chat_history: list, latest_user_image_description: str) -> None:
|
||||
if not session_id:
|
||||
return
|
||||
mp = manifest_path(session_id)
|
||||
turns_out: list[dict] = []
|
||||
ch = chat_history or []
|
||||
i = 0
|
||||
while i < len(ch):
|
||||
if ch[i].get("role") != "user":
|
||||
i += 1
|
||||
continue
|
||||
user_msg = ch[i]
|
||||
assistant_msg = ch[i + 1] if i + 1 < len(ch) and ch[i + 1].get("role") == "assistant" else None
|
||||
|
||||
turns_out.append(
|
||||
{
|
||||
"user": _serialize_user_msg_for_manifest(user_msg),
|
||||
"assistant": _serialize_assistant_entry(assistant_msg),
|
||||
}
|
||||
)
|
||||
i += 2 if assistant_msg else 1
|
||||
|
||||
os.makedirs(os.path.dirname(mp), exist_ok=True)
|
||||
with open(mp, "w", encoding="utf-8") as f:
|
||||
json.dump({"version": 2, "turns": turns_out}, f, ensure_ascii=False, indent=2)
|
||||
|
||||
def _assistant_manifest_entry_to_content(entry: dict) -> str | list:
|
||||
text = (entry.get("text") or "").strip()
|
||||
output_type = entry.get("output_type") or "text"
|
||||
autoplay = bool(entry.get("autoplay"))
|
||||
audio_duration_seconds = entry.get("audio_duration_seconds")
|
||||
audio_relpath = entry.get("audio_relpath")
|
||||
audio_abs_path = None
|
||||
if audio_relpath:
|
||||
root_norm = os.path.normpath(MULTI_INPUTS_ROOT)
|
||||
audio_abs_path = os.path.normpath(os.path.join(root_norm, audio_relpath.replace("/", os.sep)))
|
||||
try:
|
||||
under = os.path.commonpath([audio_abs_path, root_norm]) == root_norm
|
||||
except ValueError:
|
||||
under = False
|
||||
if not under or not os.path.isfile(audio_abs_path):
|
||||
audio_abs_path = None
|
||||
parts: list[dict] = []
|
||||
if output_type in ("text", "both") and text:
|
||||
parts.append({"type": "text", "text": text})
|
||||
if output_type in ("audio", "both") and audio_abs_path:
|
||||
parts.append({
|
||||
"type": "audio",
|
||||
"audio_path": audio_abs_path,
|
||||
"path": audio_abs_path,
|
||||
"autoplay": autoplay,
|
||||
"duration_seconds": audio_duration_seconds,
|
||||
})
|
||||
if not parts:
|
||||
return text
|
||||
if len(parts) == 1 and parts[0].get("type") == "text":
|
||||
return parts[0]["text"]
|
||||
return parts
|
||||
|
||||
|
||||
def _load_ui_messages_from_manifest(session_id: str) -> list | None:
|
||||
mp = manifest_path(session_id)
|
||||
if not os.path.isfile(mp):
|
||||
return None
|
||||
try:
|
||||
with open(mp, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
if "turns" not in data:
|
||||
return None
|
||||
turns = data.get("turns") or []
|
||||
if turns == []:
|
||||
return []
|
||||
out: list[dict] = []
|
||||
n = len(turns)
|
||||
for idx, t in enumerate(turns):
|
||||
out.append({"role": "user", "content": _manifest_user_entry_to_content(t.get("user") or {})})
|
||||
asst_entry = t.get("assistant") or {}
|
||||
if not isinstance(asst_entry, dict):
|
||||
asst_entry = {"text": str(asst_entry)}
|
||||
asst_content = _assistant_manifest_entry_to_content(asst_entry)
|
||||
if idx == n - 1 and not str(asst_entry.get("text") or "").strip() and asst_content == "":
|
||||
continue
|
||||
|
||||
# 从扁平存储重建与 execute_agent 产物相同的 meta 字典
|
||||
meta = {
|
||||
"output_type": asst_entry.get("output_type", "text"),
|
||||
"autoplay": asst_entry.get("autoplay", False),
|
||||
}
|
||||
audio_relpath = asst_entry.get("audio_relpath")
|
||||
if audio_relpath:
|
||||
root_norm = os.path.normpath(MULTI_INPUTS_ROOT)
|
||||
audio_path = os.path.normpath(os.path.join(root_norm, audio_relpath.replace("/", os.sep)))
|
||||
if os.path.isfile(audio_path):
|
||||
meta["audio"] = {
|
||||
"path": audio_path,
|
||||
"autoplay": asst_entry.get("autoplay", False),
|
||||
"duration_seconds": asst_entry.get("audio_duration_seconds"),
|
||||
"format": "wav",
|
||||
"sample_rate": 24000,
|
||||
}
|
||||
out.append({"role": "assistant", "content": asst_content, "meta": meta})
|
||||
return out
|
||||
|
||||
|
||||
def load_ui_messages_for_session(session_id: str) -> list:
|
||||
if not session_id:
|
||||
return []
|
||||
m = _load_ui_messages_from_manifest(session_id)
|
||||
if m is not None:
|
||||
return m
|
||||
return _load_ui_messages_from_db(session_id)
|
||||
|
||||
|
||||
def rewrite_user_files_for_last_turn(chat_history: list, session_id: str) -> None:
|
||||
if session_id and chat_history and chat_history[-1].get("role") == "user":
|
||||
rewrite_user_message_files_to_session_dir(chat_history[-1], session_id)
|
||||
|
||||
|
||||
def add_message(chat_history, query):
|
||||
chat_history = chat_history or []
|
||||
user_content = _build_user_chat_content(query)
|
||||
if user_content is None:
|
||||
return chat_history, None
|
||||
chat_history.append({"role": "user", "content": user_content})
|
||||
return chat_history, {"text": "", "files": []}
|
||||
|
||||
|
||||
def clear_ui_and_manifest(session_id: str):
|
||||
if session_id:
|
||||
mp = manifest_path(session_id)
|
||||
os.makedirs(os.path.dirname(mp), exist_ok=True)
|
||||
with open(mp, "w", encoding="utf-8") as f:
|
||||
json.dump({"version": 1, "turns": []}, f, ensure_ascii=False, indent=2)
|
||||
return [], {"text": "", "files": []}
|
||||
Reference in new issue
Block a user