commit 110fe780167d065628be142fc2db29880b96616d Author: minorQAQ <202200120184@mail.sdu.edu.cn> Date: Fri May 15 16:05:32 2026 +0800 qwenasr-qwen36-voxcmp2 base model. -2026/05/14 night diff --git a/.env b/.env new file mode 100644 index 0000000..01284ab --- /dev/null +++ b/.env @@ -0,0 +1,26 @@ +LANGSMITH_TRACING=True +LANGSMITH_ENDPOINT=https://api.smith.langchain.com +LANGSMITH_API_KEY=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 +LANGSMITH_PROJECT="agent_learning" +DATABASE_URI=sqlite:///./langgraph.db +REDIS_URI=redis://localhost:6379/0 +#spark： +# LLM_API_KEY='semi-ai.tech' +# LLM_BASE_URL="http://192.168.137.96:8000/v1" +# LLM_MODEL = "Qwen/Qwen3.6-35B-A3B-FP8" +ASR_TTS_BASE_URL = "http://192.168.137.96:8100" + +#6000pro： +LLM_API_KEY='semi-ai.tech' +LLM_BASE_URL="http://semi-ai.tech:40056/v1" +LLM_MODEL = "Intel/Qwen3.6-27B-int4-AutoRound" + + +#A100： +# LLM_API_KEY = 'sk-kGhWAQTqLk9ECsmIDb758a456bD34e92B9771dF2E36432Fb' +# LLM_BASE_URL = "https://app.semi-ai.tech:60004/v1" +# LLM_MODEL = "Qwen3.5-122B-A10B" +# LLM_TIMEOUT = 300 +# LLM_REQUESTS_PER_SECOND = 10.0 +# LLM_MAX_BUCKET_SIZE = 10 +# LLM_CHECK_EVERY_N_SECONDS = 0.1 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..29e8959 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Required LLM configuration +LLM_API_KEY='dummy' +LLM_BASE_URL="http://172.16.9.73:8000/v1" +LLM_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8" +LLM_REQUESTS_PER_SECOND = 10.0 +LLM_MAX_BUCKET_SIZE = 10 +LLM_CHECK_EVERY_N_SECONDS = 0.1 + +# Optional LangSmith configuration +LANGSMITH_TRACING=True +LANGSMITH_ENDPOINT=https://api.smith.langchain.com +LANGSMITH_API_KEY=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 +LANGSMITH_PROJECT="agent_learning" + + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0fa9caa --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +src/agent/history/*.db diff --git a/.langgraph_api/.langgraph_checkpoint.1.pckl b/.langgraph_api/.langgraph_checkpoint.1.pckl new file mode 100644 index 0000000..6420913 Binary files /dev/null and b/.langgraph_api/.langgraph_checkpoint.1.pckl differ diff --git a/.langgraph_api/.langgraph_checkpoint.2.pckl b/.langgraph_api/.langgraph_checkpoint.2.pckl new file mode 100644 index 0000000..30778a1 Binary files /dev/null and b/.langgraph_api/.langgraph_checkpoint.2.pckl differ diff --git a/.langgraph_api/.langgraph_checkpoint.3.pckl b/.langgraph_api/.langgraph_checkpoint.3.pckl new file mode 100644 index 0000000..f0c698a Binary files /dev/null and b/.langgraph_api/.langgraph_checkpoint.3.pckl differ diff --git a/.langgraph_api/.langgraph_ops.pckl b/.langgraph_api/.langgraph_ops.pckl new file mode 100644 index 0000000..ccde20a Binary files /dev/null and b/.langgraph_api/.langgraph_ops.pckl differ diff --git a/.langgraph_api/.langgraph_retry_counter.pckl b/.langgraph_api/.langgraph_retry_counter.pckl new file mode 100644 index 0000000..a3e8c9c Binary files /dev/null and b/.langgraph_api/.langgraph_retry_counter.pckl differ diff --git a/.langgraph_api/store.pckl b/.langgraph_api/store.pckl new file mode 100644 index 0000000..0a47446 Binary files /dev/null and b/.langgraph_api/store.pckl differ diff --git a/.langgraph_api/store.vectors.pckl b/.langgraph_api/store.vectors.pckl new file mode 100644 index 0000000..0a47446 Binary files /dev/null and b/.langgraph_api/store.vectors.pckl differ diff --git a/README.md b/README.md new file mode 100644 index 0000000..27eb401 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +python -m venv myenv +source myenv/bin/activate || myenv/Scripts/activate +python -m pip install --upgrade pip +pip install -U langchain langchain-openai dotenv +pip install -e .(详情见pyproject.toml) +pip install langgraph-cli[inmem] +pip install sqlalchemy pymysql loguru psycopg2-binary(postgresql) *可选数据库开发 + +# Environment +Copy `.env.example` to `.env` and fill in the required values before running the app. +cp .env.example .env + +# Required variables +# - LLM_BASE_URL +# - LLM_API_KEY +# - LLM_MODEL + +# Web 对话（推荐）：在 ``Agent`` 目录下保证 ``PYTHONPATH`` 包含 ``src`` 后执行 +cd ./Agent +uvicorn agent.web.server:app --host 127.0.0.1 --port 8765 + +# 发送消息为两阶段 API：``POST /api/chat/start``（立即落盘用户消息）+ ``POST /api/chat/complete``（推理）；前端已按此顺序调用。 +cloudflared tunnel --url http://localhost:8765开公网 + + diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/__pycache__/__init__.cpython-312.pyc b/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..4753297 Binary files /dev/null and b/__pycache__/__init__.cpython-312.pyc differ diff --git a/agent.egg-info/PKG-INFO b/agent.egg-info/PKG-INFO new file mode 100644 index 0000000..884735a --- /dev/null +++ b/agent.egg-info/PKG-INFO @@ -0,0 +1,39 @@ +Metadata-Version: 2.4 +Name: agent +Version: 0.0.1 +Summary: Starter template for making a new agent LangGraph. +Author-email: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> +License: MIT +Requires-Python: >=3.10 +Description-Content-Type: text/markdown +Requires-Dist: langgraph>=1.0.0 +Requires-Dist: python-dotenv>=1.0.1 +Requires-Dist: requests>=2.31.0 +Requires-Dist: langchain>=0.1.0 +Requires-Dist: langchain-core>=0.1.0 +Requires-Dist: langchain-community>=0.1.0 +Requires-Dist: langchain-openai>=0.2.0 +Requires-Dist: openai>=1.0.0 +Requires-Dist: langchain-qwq>=0.1.0 +Provides-Extra: dev +Requires-Dist: mypy>=1.11.1; extra == "dev" +Requires-Dist: ruff>=0.6.1; extra == "dev" + +python -m venv myenv +source myenv/bin/activate || myenv/Scripts/activate +python -m pip install --upgrade pip +pip install -U langchain langchain-openai dotenv +pip install -e .(详情见pyproject.toml) +pip install langgraph-cli[inmem] +pip install sqlalchemy pymysql loguru psycopg2-binary(postgresql) *可选数据库开发 + +# Environment +Copy `.env.example` to `.env` and fill in the required values before running the app. +cp .env.example .env + +# Required variables +# - LLM_BASE_URL +# - LLM_API_KEY +# - LLM_MODEL + +# Start with command 'langgraph dev' diff --git a/agent.egg-info/SOURCES.txt b/agent.egg-info/SOURCES.txt new file mode 100644 index 0000000..bd3db3a --- /dev/null +++ b/agent.egg-info/SOURCES.txt @@ -0,0 +1,16 @@ +README.md +pyproject.toml +agent.egg-info/PKG-INFO +agent.egg-info/SOURCES.txt +agent.egg-info/dependency_links.txt +agent.egg-info/requires.txt +agent.egg-info/top_level.txt +src/agent/__init__.py +src/agent/agent.py +src/agent/chatbot_agent.py +src/agent/llm.py +src/agent/sql_agent.py +src/agent/utils/__init__.py +src/agent/utils/database_utils.py +src/agent/utils/env_utils.py +src/agent/utils/log_utils.py \ No newline at end of file diff --git a/agent.egg-info/dependency_links.txt b/agent.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/agent.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/agent.egg-info/requires.txt b/agent.egg-info/requires.txt new file mode 100644 index 0000000..e15ad46 --- /dev/null +++ b/agent.egg-info/requires.txt @@ -0,0 +1,13 @@ +langgraph>=1.0.0 +python-dotenv>=1.0.1 +requests>=2.31.0 +langchain>=0.1.0 +langchain-core>=0.1.0 +langchain-community>=0.1.0 +langchain-openai>=0.2.0 +openai>=1.0.0 +langchain-qwq>=0.1.0 + +[dev] +mypy>=1.11.1 +ruff>=0.6.1 diff --git a/agent.egg-info/top_level.txt b/agent.egg-info/top_level.txt new file mode 100644 index 0000000..2917b34 --- /dev/null +++ b/agent.egg-info/top_level.txt @@ -0,0 +1,3 @@ +agent +langgraph +utils diff --git a/description.txt b/description.txt new file mode 100644 index 0000000..729d1a2 --- /dev/null +++ b/description.txt @@ -0,0 +1 @@ +目前版本已实现多模态聊天加tool calling \ No newline at end of file diff --git a/langgraph.json b/langgraph.json new file mode 100644 index 0000000..d6cc441 --- /dev/null +++ b/langgraph.json @@ -0,0 +1,9 @@ +{ + "dependencies": [ + "." + ], + "graphs": { + "agent": "./src/agent/agent.py:agent" + }, + "env": ".env" +} \ No newline at end of file diff --git a/llm_server/README.md b/llm_server/README.md new file mode 100644 index 0000000..7380c7b --- /dev/null +++ b/llm_server/README.md @@ -0,0 +1,35 @@ +This bundle is self-contained for running `Qwen/Qwen3.6-35B-A3B-FP8` with +`local/vllm-qwen36-nvfp4:v0.19.1-cu130` on another Spark machine. + +Contents: +- `image/`: Docker image tarball +- `modelscope/`: bundled ModelScope cache for the model weights +- `chat_template.jinja`: chat template mounted into the container +- `start_vllm.sh`: loads the image if needed and starts the container +- `stop_vllm.sh`: stops the container + +Usage after extraction: + +```bash +cd vllm-qwen36-bundle +chmod +x start_vllm.sh stop_vllm.sh +./start_vllm.sh +``` + +Optional overrides: +- `HOST_PORT=8001 ./start_vllm.sh` +- `CONTAINER_NAME=my-vllm ./start_vllm.sh` + +The startup script recreates the `Qwen/Qwen3.6-35B-A3B-FP8` symlink inside the +bundled ModelScope cache so the bundle remains portable across machines. + + +*************audio************* +# 前台运行，Ctrl+C 直接停止 +./start_audio.sh + +# 后台运行 +nohup ./start_audio.sh > /dev/null 2>&1 & + +# 后台运行时用 stop 停止 +./stop_audio.sh diff --git a/llm_server/chat_template.jinja b/llm_server/chat_template.jinja new file mode 100644 index 0000000..a6dee90 --- /dev/null +++ b/llm_server/chat_template.jinja @@ -0,0 +1,123 @@ +{%- set enable_thinking = enable_thinking | default(false) %} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- messages[0].content }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- "<|audio_start|><|audio_pad|><|audio_end|>" }} + {%- elif content.type == 'video' or 'video' in content %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif content.type == 'text' %} + {{- content.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} + {{- '\n\n' }} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- if messages[0].content is string %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {%- for content in messages[0].content %} + {%- if content.type == 'image' or 'image' in content or 'image_url' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|image_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'audio' or 'audio' in content or 'audio_url' in content %} + {{- '<|im_start|>system\n' +"<|audio_start|><|audio_pad|><|audio_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'video' or 'video' in content %} + {{- '<|im_start|>system\n' +"<|vision_start|><|video_pad|><|vision_end|>"+ '<|im_end|>\n' }} + {%- elif content.type == 'text' %} + {{- '<|im_start|>system\n' +content.text+ '<|im_end|>\n' }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = namespace(text="") %} + {%- for mcontent in message.content %} + {%- if mcontent.type == 'image' or 'image' in mcontent or 'image_url' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|image_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'audio' or 'audio' in mcontent or 'audio_url' in mcontent %} + {%- set content.text = content.text~"<|audio_start|><|audio_pad|><|audio_end|>" %} + {%- elif mcontent.type == 'video' or 'video' in mcontent %} + {%- set content.text = content.text~"<|vision_start|><|video_pad|><|vision_end|>" %} + {%- elif mcontent.type == 'text' %} + {%- set content.text = content.text~mcontent.text %} + {%- endif %} + {%- endfor %} + {%- set content = content.text %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = "" %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip("\n") + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %}{{- '\n' }}{%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}{{- '<|im_start|>user' }}{%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}{{- '<|im_end|>\n' }}{%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %}{{- '\n\n\n\n' }}{%- endif %} +{%- endif %} diff --git a/llm_server/speech_server.py b/llm_server/speech_server.py new file mode 100644 index 0000000..c06a74b --- /dev/null +++ b/llm_server/speech_server.py @@ -0,0 +1,87 @@ +from fastapi import FastAPI, UploadFile, Query +from fastapi.responses import Response +import io +import numpy as np +import soundfile as sf +from contextlib import asynccontextmanager +import torch +import tempfile +import os + +# 直接指定本地模型文件夹（绝对路径） +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +ASR_MODEL_PATH = os.path.join(BASE_DIR, "modelscope", "hub", "models", "Qwen", "Qwen3-ASR-1___7B") +TTS_MODEL_PATH = os.path.join(BASE_DIR, "modelscope", "hub", "models", "OpenBMB", "VoxCPM2") + +models = {} + +@asynccontextmanager +async def lifespan(app: FastAPI): + from transformers import BitsAndBytesConfig + quantization_asr_config = BitsAndBytesConfig(load_in_8bit=True) + + print(f"Loading ASR model from {ASR_MODEL_PATH} ...") + from qwen_asr import Qwen3ASRModel + models["asr"] = Qwen3ASRModel.from_pretrained( + ASR_MODEL_PATH, + dtype=torch.bfloat16, + device_map="cuda:0", + max_inference_batch_size=4, + attn_implementation="flash_attention_2", + max_new_tokens=512, + quantization_config=quantization_asr_config + ) + + from voxcpm import VoxCPM + models["tts"] = VoxCPM.from_pretrained( + TTS_MODEL_PATH, + load_denoiser=False, + ) + models["tts_sample_rate"] = models["tts"].tts_model.sample_rate + + print("ASR & TTS models loaded.") + yield + del models["asr"], models["tts"] + +app = FastAPI(lifespan=lifespan) + +@app.post("/asr") +async def speech_to_text(file: UploadFile): + # 保存上传文件为临时 WAV + suffix = os.path.splitext(file.filename)[-1] or ".wav" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: + tmp.write(await file.read()) + tmp_path = tmp.name + + try: + # 调用 transcribe，这里直接使用自动语言检测 + results = models["asr"].transcribe( + audio=[tmp_path], + language=None, # 自动检测，或指定 ["Chinese", "English"] + return_time_stamps=False, # 如需时间戳可改为 True + ) + # 取第一个结果（仅一个音频） + if results: + text = results[0].text + else: + text = "" + finally: + os.unlink(tmp_path) # 清理临时文件 + + return {"text": text} + +@app.get("/tts") +async def text_to_speech(text: str = Query(...)): + with torch.no_grad(): + wav = models["tts"].generate( + text=text, + cfg_value=2.0, + inference_timesteps=10, + ) + buffer = io.BytesIO() + sf.write(buffer, wav, models["tts_sample_rate"], format="WAV") + return Response(buffer.getvalue(), media_type="audio/wav") + +@app.get("/health") +async def health(): + return {"status": "ok", "models": list(models.keys())} diff --git a/llm_server/start_asr.sh b/llm_server/start_asr.sh new file mode 100644 index 0000000..e0e8764 --- /dev/null +++ b/llm_server/start_asr.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +# 配置区域（可根据实际情况修改） +MODEL_BASE_PATH="${MODEL_BASE_PATH:-/mnt/Qwen3-Omni-30B-A3B-Instruct-NVFP4}" +THINKER_PATH="${MODEL_BASE_PATH}/thinker" +CONTAINER_NAME="${CONTAINER_NAME:-vllm-thinker}" +HOST_PORT="${HOST_PORT:-8091}" +GPU_DEVICES="${GPU_DEVICES:-all}" # 例如 "0,1" 或 "all" +TENSOR_PARALLEL_SIZE="${TENSOR_PARALLEL_SIZE:-1}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}" +VLLM_IMAGE="${VLLM_IMAGE:-vllm/vllm-openai:latest}" + +# 检查模型路径是否存在 +if [[ ! -d "$THINKER_PATH" ]]; then + echo "错误: 模型路径不存在: $THINKER_PATH" + echo "请设置环境变量 MODEL_BASE_PATH 指向正确的模型根目录（包含 thinker 子目录）" + exit 1 +fi + +# 检查 Docker 可用性 +if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + DOCKER_CMD="docker" +else + DOCKER_CMD="sudo docker" +fi + +# 停止并移除可能已存在的同名容器 +${DOCKER_CMD} rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + +echo "启动 Thinker 服务..." +echo "模型路径: $THINKER_PATH" +echo "服务端口: $HOST_PORT" +echo "Tensor Parallel Size: $TENSOR_PARALLEL_SIZE" + +# 启动容器 +${DOCKER_CMD} run -d \ + --name "${CONTAINER_NAME}" \ + --gpus "${GPU_DEVICES}" \ + --ipc=host \ + --network host \ + -v "${THINKER_PATH}:/model:ro" \ + -e VLLM_USE_MODELSCOPE=false \ + "${VLLM_IMAGE}" \ + sh -c "pip install --no-cache-dir nvidia-modelopt && \ + vllm serve /model \ + --port ${HOST_PORT} \ + --quantization modelopt_fp4 \ + --dtype bfloat16 \ + --max-model-len ${MAX_MODEL_LEN} \ + --tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \ + --trust-remote-code \ + --enforce-eager" + +# 等待服务启动 +sleep 3 + +# 健康检查 +echo "健康检查 (等待几秒后):" +echo " curl http://127.0.0.1:${HOST_PORT}/health" +echo "查看日志:" +echo " ${DOCKER_CMD} logs -f ${CONTAINER_NAME}" \ No newline at end of file diff --git a/llm_server/start_audio.sh b/llm_server/start_audio.sh new file mode 100644 index 0000000..8462f11 --- /dev/null +++ b/llm_server/start_audio.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUNDLE_DIR="${SCRIPT_DIR}" +VENV_DIR="${BUNDLE_DIR}/.venv_audio" +HOST_PORT="${HOST_PORT:-8100}" + +# 激活虚拟环境 +source "${VENV_DIR}/bin/activate" + +# 检查端口占用 +if lsof -i "tcp:${HOST_PORT}" -t >/dev/null 2>&1; then + echo "Port ${HOST_PORT} already in use" >&2 + exit 1 +fi + +echo "Starting audio server on port ${HOST_PORT} ..." +cd "${BUNDLE_DIR}" +export MODELSCOPE_DOWNLOAD_MODE=local +export MODELSCOPE_HUB_ENABLE_FILE_CHECK=false +export MODELSCOPE_CACHE="${BUNDLE_DIR}/modelscope" +# 强制 HuggingFace transformers 离线 +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 +uvicorn speech_server:app --host 0.0.0.0 --port "${HOST_PORT}" --log-level info diff --git a/llm_server/start_tts.sh b/llm_server/start_tts.sh new file mode 100644 index 0000000..e69de29 diff --git a/llm_server/start_vllm.sh b/llm_server/start_vllm.sh new file mode 100644 index 0000000..13a6a89 --- /dev/null +++ b/llm_server/start_vllm.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUNDLE_DIR="${SCRIPT_DIR}" + +IMAGE_TAG="local/vllm-qwen36-nvfp4:v0.19.1-cu130" +IMAGE_TAR="${BUNDLE_DIR}/image/vllm-qwen36-nvfp4_v0.19.1-cu130.tar" +MODELSCOPE_DIR="${BUNDLE_DIR}/modelscope" +CHAT_TEMPLATE="${BUNDLE_DIR}/chat_template.jinja" +CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-nvfp4}" +HOST_PORT="${HOST_PORT:-8000}" + +if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + DOCKER=(docker) +else + DOCKER=(sudo docker) +fi + +require_path() { + local path="$1" + if [[ ! -e "${path}" ]]; then + echo "Missing required path: ${path}" >&2 + exit 1 + fi +} + +require_path "${IMAGE_TAR}" +require_path "${MODELSCOPE_DIR}" +require_path "${CHAT_TEMPLATE}" + +mkdir -p "${MODELSCOPE_DIR}/hub/models/Qwen" + +MODEL_REAL_DIR="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3___6-35B-A3B-FP8" +MODEL_LINK="${MODELSCOPE_DIR}/hub/models/Qwen/Qwen3.6-35B-A3B-FP8" + +require_path "${MODEL_REAL_DIR}" + +rm -f "${MODEL_LINK}" +ln -s "Qwen3___6-35B-A3B-FP8" "${MODEL_LINK}" + +if ! "${DOCKER[@]}" image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then + echo "Loading image ${IMAGE_TAG} from ${IMAGE_TAR}" + "${DOCKER[@]}" load -i "${IMAGE_TAR}" +fi + +"${DOCKER[@]}" rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true + +echo "Starting container ${CONTAINER_NAME} on port ${HOST_PORT}" +"${DOCKER[@]}" run -d \ + --name "${CONTAINER_NAME}" \ + --gpus all \ + --ipc=host \ + --network host \ + -e VLLM_USE_MODELSCOPE=true \ + -v "${MODELSCOPE_DIR}:/root/.cache/modelscope" \ + -v "${MODELSCOPE_DIR}:/home/liyh/.cache/modelscope" \ + -v "${CHAT_TEMPLATE}:/root/chat_template.jinja:ro" \ + "${IMAGE_TAG}" \ + Qwen/Qwen3.6-35B-A3B-FP8 \ + --port "${HOST_PORT}" \ + --tensor-parallel-size 1 \ + --max-model-len 131072 \ + --reasoning-parser qwen3 \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --api-key semi-ai.tech \ + --enable-prefix-caching \ + --max_num_batched_tokens 4096 \ + --max_num_seqs 8 \ + --kv-cache-dtype fp8_e4m3 \ + --chat-template /root/chat_template.jinja \ + --safetensors-load-strategy eager \ + --gpu-memory-utilization 0.75\ + "$@" + +echo +echo "Health check:" +echo " curl http://127.0.0.1:${HOST_PORT}/health" +echo "Logs:" +echo " ${DOCKER[*]} logs -f ${CONTAINER_NAME}" diff --git a/llm_server/stop_asr.sh b/llm_server/stop_asr.sh new file mode 100644 index 0000000..3bae62e --- /dev/null +++ b/llm_server/stop_asr.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER_NAME="${CONTAINER_NAME:-vllm-thinker}" + +if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + DOCKER_CMD="docker" +else + DOCKER_CMD="sudo docker" +fi + +echo "停止并移除容器: $CONTAINER_NAME" +${DOCKER_CMD} rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true +echo "已停止" \ No newline at end of file diff --git a/llm_server/stop_audio.sh b/llm_server/stop_audio.sh new file mode 100644 index 0000000..b074b85 --- /dev/null +++ b/llm_server/stop_audio.sh @@ -0,0 +1,14 @@ +#!usrbinenv bash +set -euo pipefail + +HOST_PORT=${HOST_PORT-8100} + +PID=$(lsof -i tcp${HOST_PORT} -t 2devnull true) + +if [[ -z ${PID} ]]; then + echo No process found on port ${HOST_PORT} + exit 0 +fi + +echo Stopping audio server (PID ${PID}) ... +kill ${PID} \ No newline at end of file diff --git a/llm_server/stop_tts.sh b/llm_server/stop_tts.sh new file mode 100644 index 0000000..e69de29 diff --git a/llm_server/stop_vllm.sh b/llm_server/stop_vllm.sh new file mode 100644 index 0000000..6eea4ad --- /dev/null +++ b/llm_server/stop_vllm.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER_NAME="${CONTAINER_NAME:-vllm-qwen36-nvfp4}" + +if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + DOCKER=(docker) +else + DOCKER=(sudo docker) +fi + +"${DOCKER[@]}" rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true +echo "Stopped ${CONTAINER_NAME}" diff --git a/llm_server/test.py b/llm_server/test.py new file mode 100644 index 0000000..501aab2 --- /dev/null +++ b/llm_server/test.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +语音服务功能测试：ASR 识别 + TTS 合成 +""" +import requests +import json +import os +import sys +import time + +HOST = "http://127.0.0.1:8100" + +def test_health(): + """1. 健康检查""" + print("=" * 50) + print("📋 健康检查") + print("=" * 50) + resp = requests.get(f"{HOST}/health") + print(f"状态码: {resp.status_code}") + print(f"模型列表: {resp.json()}") + assert resp.status_code == 200 + assert "asr" in resp.json()["models"] + assert "tts" in resp.json()["models"] + print("✅ 健康检查通过\n") + +def test_asr(audio_path=None): + """2. 测试语音识别""" + print("=" * 50) + print("🎤 语音识别测试") + print("=" * 50) + + if audio_path is None: + # 如果没有本地音频，下载一个公开示例 + import tempfile + audio_path = tempfile.mktemp(suffix=".wav") + url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_zh.wav" + print(f"下载测试音频: {url}") + import subprocess + subprocess.run(["wget", "-q", url, "-O", audio_path], check=True) + + print(f"音频文件: {audio_path}") + with open(audio_path, "rb") as f: + resp = requests.post(f"{HOST}/asr", files={"file": f}) + + print(f"状态码: {resp.status_code}") + result = resp.json() + print(f"识别结果: {result.get('text', 'N/A')}") + assert resp.status_code == 200 + assert len(result.get("text", "")) > 0 + print("✅ 语音识别通过\n") + return result["text"] + +def test_tts(text="(激动地说)你好世界，这是一段测试语音", output="test_output.wav"): + """3. 测试语音合成""" + print("=" * 50) + print("🔊 语音合成测试") + print("=" * 50) + print(f"合成文本: {text}") + + start_time = time.time() + resp = requests.get(f"{HOST}/tts", params={"text": text}) + elapsed = time.time() - start_time + + print(f"状态码: {resp.status_code}") + print(f"耗时: {elapsed:.2f}秒") + + if resp.status_code == 200: + with open(output, "wb") as f: + f.write(resp.content) + file_size = os.path.getsize(output) / 1024 # KB + print(f"音频已保存: {output} ({file_size:.1f} KB)") + else: + print(f"错误: {resp.text}") + + assert resp.status_code == 200 + assert os.path.exists(output) + print("✅ 语音合成通过\n") + +def test_asr_english(): + """4. 测试英文识别""" + print("=" * 50) + print("🌍 英文识别测试") + print("=" * 50) + + import tempfile + audio_path = tempfile.mktemp(suffix=".wav") + url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav" + print(f"下载英文测试音频: {url}") + import subprocess + subprocess.run(["wget", "-q", url, "-O", audio_path], check=True) + + with open(audio_path, "rb") as f: + resp = requests.post(f"{HOST}/asr", files={"file": f}) + + print(f"状态码: {resp.status_code}") + result = resp.json() + print(f"识别结果: {result.get('text', 'N/A')}") + assert resp.status_code == 200 + assert len(result.get("text", "")) > 0 + print("✅ 英文识别通过\n") + +def test_tts_long_text(): + """5. 测试长文本合成""" + print("=" * 50) + print("📜 长文本合成测试") + print("=" * 50) + + long_text = """人工智能是计算机科学的一个分支，它企图了解智能的实质， + 并生产出一种新的能以人类智能相似的方式做出反应的智能机器， + 该领域的研究包括机器人、语言识别、图像识别、自然语言处理和专家系统等。""" + + print(f"文本长度: {len(long_text)} 字") + start_time = time.time() + resp = requests.get(f"{HOST}/tts", params={"text": long_text}) + elapsed = time.time() - start_time + + print(f"状态码: {resp.status_code}") + print(f"耗时: {elapsed:.2f}秒") + + if resp.status_code == 200: + output = "test_long_output.wav" + with open(output, "wb") as f: + f.write(resp.content) + file_size = os.path.getsize(output) / 1024 + print(f"音频已保存: {output} ({file_size:.1f} KB)") + + assert resp.status_code == 200 + print("✅ 长文本合成通过\n") + +if __name__ == "__main__": + print("\n" + "🚀 开始测试语音服务".center(50, "=")) + print(f"服务地址: {HOST}\n") + + try: + test_health() + + # 可选：用自己的音频文件测试 + # test_asr("/path/to/your/audio.wav") + test_asr() + test_asr_english() + + test_tts() + test_tts_long_text() + + print("\n" + "🎉 所有测试通过！".center(50, "=")) + except Exception as e: + print(f"\n❌ 测试失败: {e}") + sys.exit(1) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..df5644b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,77 @@ +[project] +name = "agent" +version = "0.0.1" +description = "Starter template for making a new agent LangGraph." +authors = [ + { name = "William Fu-Hinthorn", email = "13333726+hinthornw@users.noreply.github.com" }, +] +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +dependencies = [ + "langgraph>=1.0.0", + "python-dotenv>=1.0.1", + "requests>=2.31.0", + "langchain>=0.1.0", + "langchain-core>=0.1.0", + "langchain-community>=0.1.0", + "langchain-openai>=0.2.0", + "openai>=1.0.0", + "langchain-qwq>=0.1.0", + "fastapi>=0.115.0", + "uvicorn[standard]>=0.32.0", +] + + +[project.optional-dependencies] +dev = ["mypy>=1.11.1", "ruff>=0.6.1"] + +[build-system] +requires = ["setuptools>=73.0.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["langgraph.templates.agent", "agent", "utils"] +[tool.setuptools.package-dir] +"langgraph.templates.agent" = "src/agent" +"agent" = "src/agent" +"utils" = "src/agent/utils" + + +[tool.setuptools.package-data] +"*" = ["py.typed"] +"agent" = ["web/**/*"] +"langgraph.templates.agent" = ["web/**/*"] + +[tool.ruff] +lint.select = [ + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "D", # pydocstyle + "D401", # First line should be in imperative mood + "T201", + "UP", +] +lint.ignore = [ + "UP006", + "UP007", + # We actually do want to import from typing_extensions + "UP035", + # Relax the convention by _not_ requiring documentation for every function parameter. + "D417", + "E501", +] +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["D", "UP"] +[tool.ruff.lint.pydocstyle] +convention = "google" + +[dependency-groups] +dev = [ + "anyio>=4.7.0", + "langgraph-cli[inmem]>=0.4.14", + "mypy>=1.13.0", + "pytest>=8.3.5", + "ruff>=0.8.2", +] \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__pycache__/__init__.cpython-312.pyc b/src/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..cc563b1 Binary files /dev/null and b/src/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/__init__.py b/src/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agent/__pycache__/__init__.cpython-312.pyc b/src/agent/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..54b9c69 Binary files /dev/null and b/src/agent/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/__pycache__/chatbot_agent.cpython-312.pyc b/src/agent/__pycache__/chatbot_agent.cpython-312.pyc new file mode 100644 index 0000000..b2353bc Binary files /dev/null and b/src/agent/__pycache__/chatbot_agent.cpython-312.pyc differ diff --git a/src/agent/__pycache__/llm.cpython-312.pyc b/src/agent/__pycache__/llm.cpython-312.pyc new file mode 100644 index 0000000..d75f5e2 Binary files /dev/null and b/src/agent/__pycache__/llm.cpython-312.pyc differ diff --git a/src/agent/chatbot_agent.py b/src/agent/chatbot_agent.py new file mode 100644 index 0000000..bd86810 --- /dev/null +++ b/src/agent/chatbot_agent.py @@ -0,0 +1,178 @@ +from typing import Any, cast, Dict + +from langchain_classic.agents import AgentExecutor, create_tool_calling_agent +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_core.runnables import RunnablePassthrough, RunnableWithMessageHistory +from langchain_core.messages import HumanMessage + +from agent.llm import Qwen36_35B_A3B_FP8_LLM +from agent.tools import get_tools +from agent.utils.agent_utils import assistant_text, get_history, normalize_user_text_content, summarize_history +from agent.web.ui_session import ( + collect_media_paths_and_query_text, + rewrite_user_message_files_to_session_dir, + session_dir, + sync_manifest_from_chat_history, +) +from agent.memory.system_prompt import MAIN_SYSTEM_PROMPT + +# ---------- 1. 提示模板---------- +prompt = ChatPromptTemplate.from_messages([ + ("system", "{system_message}"), + ("placeholder", "{chat_history}"), + MessagesPlaceholder(variable_name="human_message"), + ("placeholder", "{agent_scratchpad}"), +]) + +tools = get_tools() + +# ---------- 2. 创建工具调用 agent ---------- +tool_calling_agent = create_tool_calling_agent( + llm=Qwen36_35B_A3B_FP8_LLM.bind(tool_choice="auto"), + tools=tools, + prompt=prompt, +) + +# ---------- 3. agent executor ---------- +agent_executor = AgentExecutor( + agent=tool_calling_agent, + tools=tools, + verbose=True, + handle_parsing_errors=True, + stream_runnable=False, +) + +# ---------- 4. 带历史记录的链 ---------- +agent_with_history = RunnableWithMessageHistory( + runnable=agent_executor, + get_session_history=get_history, + input_messages_key="human_message", + history_messages_key="chat_history", +) + +# ---------- 5. 注入摘要和系统提示 ---------- +agent_with_history = ( + RunnablePassthrough.assign( + # ✅ 从输入数据中的 agent_config 获取配置 + summarize_result=lambda x: summarize_history(x["input"], x["agent_config"]) + ) + | RunnablePassthrough.assign( + input=lambda x: x["input"], + chat_history=lambda x: x["summarize_result"]["chat_history"], + system_message=lambda x: ( + MAIN_SYSTEM_PROMPT + + ( + "\n\n【此前对话摘要】\n" + x["summarize_result"]["summary"] + if x["summarize_result"]["summary"] + else "" + ) + ), + human_message=lambda x: [HumanMessage(content=x["input"])], + ) + | agent_with_history +) + + +# ---------- 6. 辅助函数：将前端 chat_history 转为 LangChain 消息列表 ---------- +def _extract_user_input_parts(user_msg: Dict[str, Any]) -> tuple[list[str], list[str], list[dict[str, Any]] | str]: + image_paths, audio_paths, query_text = collect_media_paths_and_query_text(user_msg) + + content_parts: list[dict[str, Any]] = [] + for img_path in image_paths: + content_parts.append({"type": "image_url", "image_url": {"url": img_path}}) + + for audio_path in audio_paths: + content_parts.append({ + "type": "audio", + "audio_path": audio_path, + }) + if query_text: + content_parts.append({"type": "text", "text": query_text}) + + if content_parts: + if len(content_parts) == 1 and content_parts[0].get("type") == "text": + return image_paths, audio_paths, content_parts[0]["text"] + return image_paths, audio_paths, content_parts + return image_paths, audio_paths, normalize_user_text_content(user_msg.get("content", query_text)) + +# ---------- 7. 核心执行函数 ---------- +def execute_agent( + chat_history: list[dict[str, Any]], + session_id: str = "test", + output_type: str = "text", +) -> list[dict[str, Any]]: + chat_history = chat_history or [] + + if not chat_history or chat_history[-1].get("role") != "user": + return chat_history + + if session_id: + rewrite_user_message_files_to_session_dir(chat_history[-1], session_id) + + _image_paths, _audio_paths, user_content = _extract_user_input_parts(chat_history[-1]) + + # 构建运行时配置 + config = { + "configurable": { + "session_id": session_id, + }, + "metadata": { + "output_type": output_type, + "audio_output_dir": session_dir(session_id) if session_id else None, + }, + "llm": Qwen36_35B_A3B_FP8_LLM, + } + + invoke_input = { + "input": user_content, + "agent_config": config, + } + + try: + result = cast(dict[str, Any], agent_with_history.invoke(invoke_input, config)) + + text_content = assistant_text(result.get("output", "")) + + output_type_from_config = config.get("metadata", {}).get("output_type", "text") + audio_data = Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs or {} + if audio_data.get("output_type") != output_type_from_config: + audio_data = {} + + assistant_meta = { + "output_type": output_type_from_config, + "autoplay": False, + "audio": audio_data.get("audio"), + } + + if output_type_from_config == "audio" and assistant_meta.get("audio"): + assistant_content = [] + elif output_type_from_config == "both" and assistant_meta.get("audio"): + assistant_content = text_content + else: + assistant_content = text_content if text_content else "（空回复）" + + chat_history.append({ + "role": "assistant", + "content": assistant_content, + "meta": assistant_meta, + }) + + # 用完后清空，避免影响后续无音频的请求 + Qwen36_35B_A3B_FP8_LLM.last_audio_kwargs = None + + except Exception as e: + # 异常处理保持不变 + import traceback + print("[ERROR] 调用模型时出错:") + traceback.print_exc() + err_text = "抱歉，我遇到了一些问题，请稍后再试。" + chat_history.append({"role": "assistant", "content": err_text}) + + if session_id: + sync_manifest_from_chat_history(session_id, chat_history, "") + + return chat_history + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/src/agent/history/data_base/chat_history.db b/src/agent/history/data_base/chat_history.db new file mode 100644 index 0000000..e5756b6 Binary files /dev/null and b/src/agent/history/data_base/chat_history.db differ diff --git a/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a b/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a new file mode 100644 index 0000000..ba3c2c7 Binary files /dev/null and b/src/agent/history/multi_inputs/20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a differ diff --git a/src/agent/history/multi_inputs/20260514_222254/assistant_tts_222333_0.wav b/src/agent/history/multi_inputs/20260514_222254/assistant_tts_222333_0.wav new file mode 100644 index 0000000..196fc5c Binary files /dev/null and b/src/agent/history/multi_inputs/20260514_222254/assistant_tts_222333_0.wav differ diff --git a/src/agent/history/multi_inputs/20260514_222254/b50c460b07654d568e897d92c776675f.jpg b/src/agent/history/multi_inputs/20260514_222254/b50c460b07654d568e897d92c776675f.jpg new file mode 100644 index 0000000..f8f53af Binary files /dev/null and b/src/agent/history/multi_inputs/20260514_222254/b50c460b07654d568e897d92c776675f.jpg differ diff --git a/src/agent/history/multi_inputs/20260514_222254/ui_messages.json b/src/agent/history/multi_inputs/20260514_222254/ui_messages.json new file mode 100644 index 0000000..7a6bd5c --- /dev/null +++ b/src/agent/history/multi_inputs/20260514_222254/ui_messages.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "turns": [ + { + "user": { + "parts": [ + { + "kind": "image_url", + "relpath": "20260514_222254/b50c460b07654d568e897d92c776675f.jpg" + }, + { + "kind": "audio", + "relpath": "20260514_222254/60a5fff67eaf461b979d76449a1a6224.m4a" + }, + { + "kind": "text", + "text": "请直接告诉我数量。" + } + ], + "image_description": "" + }, + "assistant": { + "text": "图片中有两只猫。", + "output_type": "both", + "autoplay": false, + "audio_duration_seconds": 2.08, + "audio_relpath": "20260514_222254/assistant_tts_222333_0.wav" + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260515_130220/ui_messages.json b/src/agent/history/multi_inputs/20260515_130220/ui_messages.json new file mode 100644 index 0000000..3c3bad0 --- /dev/null +++ b/src/agent/history/multi_inputs/20260515_130220/ui_messages.json @@ -0,0 +1,39 @@ +{ + "version": 2, + "turns": [ + { + "user": { + "content": [ + { + "type": "text", + "text": "你好" + } + ] + }, + "assistant": { + "text": "你好！很高兴为您服务。请问有什么我可以帮您的吗？", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null + } + }, + { + "user": { + "content": [ + { + "type": "text", + "text": "现在几点了" + } + ] + }, + "assistant": { + "text": "现在是2026年5月15日 13:02。", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null + } + } + ] +} \ No newline at end of file diff --git a/src/agent/llm.py b/src/agent/llm.py new file mode 100644 index 0000000..764f71a --- /dev/null +++ b/src/agent/llm.py @@ -0,0 +1,336 @@ +import io +import os +import tempfile +import wave +import pyaudio +from contextlib import closing +from typing import Any, List, Dict, Optional +from pydantic import Field +import re, json +from datetime import datetime as dt + +import requests +from langchain_core.messages import HumanMessage, AIMessage, BaseMessage, SystemMessage +from langchain_core.outputs import ChatResult, ChatGeneration +from langchain_openai import ChatOpenAI + +from agent.utils.image_utils import image_bytes_to_openai_image_url_part +from agent.utils.env_utils import LLM_API_KEY, LLM_BASE_URL, LLM_MODEL, LLM_TIMEOUT, ASR_TTS_BASE_URL +from agent.utils.agent_utils import assistant_text, normalize_user_text_content +from agent.utils.image_utils import ( + image_path_to_openai_image_url_part, + image_bytes_to_openai_image_url_part, + IMAGE_FILE_EXTENSIONS, +) +from agent.memory.system_prompt import VISION_SYSTEM_PROMPT + +class ChatQwen(ChatOpenAI): + output_type: str = Field(default="text") + stream_play: bool = Field(default=False) + last_audio_kwargs: Optional[dict] = Field(default=None) + def __init__(self, **kwargs: Any) -> None: + super().__init__( + model=kwargs.pop("model", LLM_MODEL), + api_key=kwargs.pop("api_key", LLM_API_KEY), + base_url=kwargs.pop("base_url", LLM_BASE_URL), + timeout=kwargs.pop("timeout", LLM_TIMEOUT), + max_retries=kwargs.pop("max_retries", 0), + **kwargs + ) + + @staticmethod + def play_audio_bytes(audio_bytes: bytes, chunk_size: int = 1024) -> None: + if not audio_bytes: + return + wf = wave.open(io.BytesIO(audio_bytes), 'rb') + p = pyaudio.PyAudio() + stream = p.open( + format=p.get_format_from_width(wf.getsampwidth()), + channels=wf.getnchannels(), + rate=wf.getframerate(), + output=True, + ) + data = wf.readframes(chunk_size) + while data: + stream.write(data) + data = wf.readframes(chunk_size) + stream.stop_stream() + stream.close() + p.terminate() + wf.close() + + @staticmethod + def _process_image_content(part: Dict[str, Any]) -> Dict[str, Any]: + image_type = part.get("type") + if image_type != "image_url": + return part + image_url = part.get("image_url", {}) + url = image_url.get("url", "") + if url.startswith(("http://", "https://", "data:")): + return part + if isinstance(url, str) and os.path.exists(url): + ext = os.path.splitext(url)[1].lower() + if ext in IMAGE_FILE_EXTENSIONS: + result = image_path_to_openai_image_url_part(url) + return result if result else {"type": "text", "text": "[图片处理失败]"} + if isinstance(url, bytes): + result = image_bytes_to_openai_image_url_part(url) + return result if result else {"type": "text", "text": "[图片处理失败]"} + return part + + @staticmethod + def _convert_audio_to_wav(audio_path: str) -> str: + ext = os.path.splitext(audio_path)[1].lower() + if ext == ".wav": + return audio_path + try: + from pydub import AudioSegment + except ImportError as exc: + raise RuntimeError("当前 ASR 仅支持 WAV，但未安装 pydub/ffmpeg") from exc + try: + audio = AudioSegment.from_file(audio_path) + except Exception as exc: + raise RuntimeError(f"无法读取音频文件: {exc}") from exc + fd, wav_path = tempfile.mkstemp(suffix=".wav", prefix="asr_") + os.close(fd) + try: + audio.export(wav_path, format="wav") + return wav_path + except Exception: + if os.path.exists(wav_path): + os.remove(wav_path) + raise + + def _audio_to_text(self, audio_path: str) -> str: + if not os.path.exists(audio_path): + raise FileNotFoundError(f"音频文件不存在: {audio_path}") + wav_path = self._convert_audio_to_wav(audio_path) + cleanup_wav = wav_path != audio_path + try: + with open(wav_path, "rb") as f: + resp = requests.post( + f"{ASR_TTS_BASE_URL}/asr", + files={"file": (os.path.basename(wav_path), f, "audio/wav")}, + ) + resp.raise_for_status() + return (resp.json().get("text") or "").strip() + finally: + if cleanup_wav and os.path.exists(wav_path): + os.remove(wav_path) + + def _preprocess_messages(self, messages: List[BaseMessage]) -> List[BaseMessage]: + new_messages = [] + for msg in messages: + if isinstance(msg, HumanMessage) and isinstance(msg.content, str): + text = msg.content.strip() + if text.startswith("[") and text.endswith("]"): + try: + possible_list = json.loads(text) + if isinstance(possible_list, list): + msg = HumanMessage(content=possible_list, additional_kwargs=msg.additional_kwargs) + except (json.JSONDecodeError, ValueError): + pass + if isinstance(msg, HumanMessage) and isinstance(msg.content, dict): + msg = HumanMessage(content=normalize_user_text_content(msg.content), additional_kwargs=msg.additional_kwargs) + elif isinstance(msg, HumanMessage) and isinstance(msg.content, list): + new_content: List[Dict[str, Any] | str] = [] + for part in msg.content: + if isinstance(part, dict): + if part.get("type") == "audio": + audio_path = part.get("audio_path") + if audio_path: + try: + text = self._audio_to_text(audio_path) + new_content.append({"type": "text", "text": text}) + except Exception as e: + new_content.append({"type": "text", "text": f"[语音识别失败: {e}]"}) + elif part.get("type") == "image_url": + new_content.append(self._process_image_content(part)) + elif part.get("type") == "text": + text = (part.get("text") or "").strip() + if text: + new_content.append({"type": "text", "text": text}) + elif part.get("path") or (part.get("file") or {}).get("path"): + file_path = part.get("path") or (part.get("file") or {}).get("path") + if isinstance(file_path, str) and os.path.isfile(file_path): + ext = os.path.splitext(file_path)[1].lower() + if ext in IMAGE_FILE_EXTENSIONS: + new_content.append(self._process_image_content({"type": "image_url", "image_url": {"url": file_path}})) + else: + new_content.append({"type": "text", "text": f"[用户上传了附件文件: {os.path.basename(file_path)}]"}) + else: + text = normalize_user_text_content(part) + if text: + new_content.append({"type": "text", "text": text}) + elif isinstance(part, str): + text = part.strip() + if text: + new_content.append(text) + if len(new_content) == 1 and isinstance(new_content[0], str): + msg = HumanMessage(content=new_content[0], additional_kwargs=msg.additional_kwargs) + else: + msg = HumanMessage(content=new_content, additional_kwargs=msg.additional_kwargs) + new_messages.append(msg) + return new_messages + + def _text_to_audio(self, text: str) -> bytes: + resp = requests.get(f"{ASR_TTS_BASE_URL}/tts", params={"text": text}) + resp.raise_for_status() + return resp.content + + @staticmethod + def _get_wav_duration_seconds(audio_bytes: bytes) -> float | None: + if not audio_bytes: + return None + try: + with closing(wave.open(io.BytesIO(audio_bytes), "rb")) as wf: + frame_rate = wf.getframerate() + frame_count = wf.getnframes() + if frame_rate <= 0: + return None + return round(frame_count / frame_rate, 2) + except wave.Error: + return None + + def _apply_tts(self, result: ChatResult, output_type: str, audio_output_dir: str | None = None) -> ChatResult: + if output_type not in ("audio", "both") or not audio_output_dir: + return result + os.makedirs(audio_output_dir, exist_ok=True) + for idx, gen in enumerate(result.generations): + if not isinstance(gen, ChatGeneration): + continue + msg = gen.message + if not isinstance(msg, AIMessage): + continue + text = assistant_text(msg.content) + if not text.strip(): + continue + try: + audio_bytes = self._text_to_audio(text) + timestamp = dt.now().strftime("%H%M%S") + audio_path = os.path.join(audio_output_dir, f"assistant_tts_{timestamp}_{idx}.wav") + with open(audio_path, "wb") as f: + f.write(audio_bytes) + duration_seconds = self._get_wav_duration_seconds(audio_bytes) + # 仅保存到实例属性，不再写入 additional_kwargs + if idx == 0: + self.last_audio_kwargs = { + "audio": { + "sample_rate": 24000, + "format": "wav", + "path": audio_path, + "duration_seconds": duration_seconds, + }, + "output_type": output_type, + } + if self.stream_play: + self.play_audio_bytes(audio_bytes) + if output_type == "audio": + msg.content = "" + except Exception as e: + print(f"[TTS ERROR] 合成语音失败: {e}") + # 发生错误时也清空 last_audio_kwargs，避免污染 + self.last_audio_kwargs = None + return result + + def _generate( + self, + messages: List[BaseMessage], + stop: Optional[List[str]] = None, + run_manager: Optional[Any] = None, + **kwargs: Any, + ) -> ChatResult: + metadata = {} + if run_manager: + metadata = getattr(run_manager, 'metadata', {}) + output_type = metadata.get("output_type", self.output_type) + audio_output_dir = metadata.get("audio_output_dir") + + # ✅ 移除自定义参数，防止它们传入 OpenAI API + kwargs.pop("output_type", None) + kwargs.pop("audio_output_dir", None) + + messages = self._preprocess_messages(messages) + + result = super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs) + + if output_type in ("audio", "both"): + result = self._apply_tts(result, output_type, audio_output_dir=audio_output_dir) + if output_type == "audio": + for gen in result.generations: + if isinstance(gen, ChatGeneration) and isinstance(gen.message, AIMessage): + gen.message.content = "" + return result + + def _create_chat_result(self, response, generation_info=None) -> ChatResult: + result = super()._create_chat_result(response, generation_info) + try: + dumped = response if isinstance(response, dict) else response.model_dump(exclude={"choices": {"__all__": {"message": {"parsed"}}}}) + for i, choice in enumerate(dumped.get("choices") or []): + if i >= len(result.generations): + break + lc_msg = result.generations[i].message + msg = choice.get("message") or {} + ak = lc_msg.additional_kwargs + merged = "" + for key in ("reasoning_content", "reasoning_details", "reasoning"): + merged = assistant_text(msg.get(key)) + if merged: + break + if merged: + ak.setdefault("reasoning_content", merged) + raw_content = assistant_text(msg.get("content")) + if not raw_content and merged: + lc_msg.content = merged + elif raw_content: + lc_msg.content = raw_content + content_text = lc_msg.content if isinstance(lc_msg.content, str) else "" + pattern = re.compile(r'\s*(.*?)\s*', re.DOTALL) + matches = pattern.findall(content_text) + if matches: + tool_calls = [] + for idx, match in enumerate(matches): + try: + call_data = json.loads(match) + except json.JSONDecodeError: + continue + tool_calls.append({ + "id": f"call_{idx}", + "type": "function", + "function": { + "name": call_data.get("name", ""), + "arguments": json.dumps(call_data.get("arguments", {})) + } + }) + if tool_calls: + ak["tool_calls"] = tool_calls + cleaned = pattern.sub("", content_text).strip() + lc_msg.content = cleaned if cleaned else "" + except Exception: + pass + return result + + def describe_image(self, image_path_or_url: str, prompt: str = "请详细描述这张图片的内容。") -> str: + messages = [ + SystemMessage(content=[{"type": "text", "text": VISION_SYSTEM_PROMPT}]), + HumanMessage(content=[ + {"type": "image_url", "image_url": {"url": image_path_or_url}}, + {"type": "text", "text": prompt} + ]) + ] + result = self.invoke(messages) + return result.content + + def describe_image_bytes(self, image_bytes: bytes, prompt: str = "请详细描述这张图片的内容。") -> str: + img_part = image_bytes_to_openai_image_url_part(image_bytes) + if img_part is None: + return "图片处理失败" + messages = [ + SystemMessage(content=[{"type": "text", "text": VISION_SYSTEM_PROMPT}]), + HumanMessage(content=[img_part, {"type": "text", "text": prompt}]) + ] + result = self.invoke(messages) + return result.content + + +Qwen36_35B_A3B_FP8_LLM = ChatQwen() diff --git a/src/agent/memory/__init__.py b/src/agent/memory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agent/memory/__pycache__/__init__.cpython-312.pyc b/src/agent/memory/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..318af58 Binary files /dev/null and b/src/agent/memory/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/memory/__pycache__/system_prompt.cpython-312.pyc b/src/agent/memory/__pycache__/system_prompt.cpython-312.pyc new file mode 100644 index 0000000..0bfd68a Binary files /dev/null and b/src/agent/memory/__pycache__/system_prompt.cpython-312.pyc differ diff --git a/src/agent/memory/system_prompt.py b/src/agent/memory/system_prompt.py new file mode 100644 index 0000000..b49ef18 --- /dev/null +++ b/src/agent/memory/system_prompt.py @@ -0,0 +1,68 @@ +MAIN_SYSTEM_PROMPT = """你是一个多模态助手，能够直接理解用户上传的图片、语音等内容。 + 【关键行为准则】 + - 你的默认行为是直接回答用户问题，不要调用任何工具。 + - 只有当某个信息是你**绝对无法从当前对话或上传内容中获取**，且该信息必须通过外部工具才能获得时，才允许调用工具。 + - 对于任何图片/音频输入，请始终**直接**利用你的视觉/听觉能力回答，**绝对不要**调用截图、OCR、日期查询等无关工具。 + - 如果你不确定是否需要工具，选择**不调用**，直接给出你能给出的最佳答案。 + - 违反上述规则是一种严重错误。 +""" + +SUMMARY_SYSTEM_PROMPT =""" + 你是一个名为“Summary Agent”的专业摘要助手，专门负责对用户提供的文本内容进行高度凝练、准确的总结。 + + ## 核心原则 + - 客观准确：严格基于原文内容，不添加个人观点、评论或外部知识。 + - 抓住要点：提取核心论点、关键数据、重要事件或结论，忽略次要细节、重复和示例。 + - 语言简洁：用清晰、简练的语句表达，避免冗长。 + - 保持原意：不歪曲原文事实和观点，即使原文存在错误或争议，仅如实概括。 + - 结构清晰：根据内容类型选择最合适的结构，如段落式或分点式，确保逻辑流畅。 + + ## 输出格式 + - 默认以一段连贯的文字呈现摘要，必要时可使用编号或项目符号。 + - 摘要长度：如果用户未指定，则将原文字数压缩至 20%~30% 左右，或直接给出约3~5句话的精简摘要。对于长文，可适当增加长度。 + - 开头不需要“摘要：”等标记，直接输出摘要内容。（如用户要求标记，可加上） + - 处理对话记录时，请总结：讨论主题、各方核心立场、达成的共识、存在的分歧及后续行动项。 + + ## 特别行为 + - 若内容不足以形成有意义的摘要（如太短或无实际信息），请回复“内容过短，无法生成有效摘要”。 + - 若用户要求总结特定方面（如仅总结“原因”或“结论”），请针对性提取，并在摘要中注明聚焦点。 + - 当需要保留原文关键术语或专有名词时，可原样引用，并补充简要解释。 + - 多语言支持：用与原文相同的语言输出摘要，除非用户指定目标语言。 + + ## 示例行为（模仿语气） + 用户：“请总结以下新闻：……（长新闻文本）……” + 你输出：“某公司于2026年5月发布了新一代量子芯片，性能较前代提升10倍，预计2027年量产。此举被业界视为量子计算商业化的关键突破，同时引发了对加密安全的讨论。” + + 现在，请以 Summary Agent 的身份，完成用户的摘要请求。 +""" + + +VISION_SYSTEM_PROMPT = """ + 你是专业的图像理解助手。 + 首先请用客观、准确的中文描述用户提供的图片，覆盖主要物体、场景、人物动作、界面元素、图表数据与清晰可读的文字； + 涉及到你认识的具体品牌、时间、地点、人物、影视剧、物品等特定内容时请直接命名说明； + 如果用户提供了多张图片，请分别编号描述(如「图1」「图2」等)，便于后续问答引用。 + 看不清的内容请如实说明，不要臆测。 + 其次请认真回答用户的问题，结合图片内容进行分析推理，给出具体、简短、清晰、实用的回复； +""" + +DB_SYSTEM_PROMT = """ + 你是一位敏捷的数据库分析助手，能够高效利用内置功能帮助用户从PostgreSQL数据库中获取答案。 + + 给定一个数据库相关问题，你需要按照以下步骤操作： + 1.创建一个语法正确的PostgreSQL查询语句，优先使用SELECT或WITH语句，确保查询只读且安全。 + 2.使用内置的SQL查询工具执行该查询，并获取结果。 + 3.将查询结果总结为一个简洁、清晰的回答，突出关键数字和关系，帮助用户理解数据背后的含义。 + + 【你的行动指南】 + 1. **永远优先使用内置功能获取真实数据**。对于任何与数据库内容相关的问题，严禁编造答案。 + 2. **智能判断调用顺序**： + - 如果用户问题模糊（例如“帮我看看用户相关的数据”），先展示所有可用表，并询问用户具体需求。 + - 如果用户指明了具体表名，可以直接查看表结构，以便构思查询。 + - 如果用户直接提供了一条看似合理的SELECT或WITH语句，可以先验证其安全性，若通过则立即执行。 + 3. **灵活处理查询结果**：当执行查询返回数据后，请将JSON结果归纳为人类友好的描述，突出关键数字和关系。若无数据返回，应如实告知并建议用户调整查询条件。 + 4. **安全底线不可突破**：无论何时，只允许执行以SELECT或WITH开头的只读查询。若功能返回安全拦截错误，必须向用户解释原因并建议修改。 + 5. **非数据库问题快速响应**：遇到问候、常识性问题或超出数据库范围的内容时，可以直接友好回应，不必强制调用功能。 + + 请记住，你是一个协作伙伴，目标是让用户以最小的学习成本从数据库中获得有价值的信息。开始工作吧！ +""" diff --git a/src/agent/tools/__init__.py b/src/agent/tools/__init__.py new file mode 100644 index 0000000..4ea8e30 --- /dev/null +++ b/src/agent/tools/__init__.py @@ -0,0 +1,32 @@ +from .tool_main import * + +# from .tool_database_class import * +# from utils.database_utils import PostgreSQL_Manager + +# 创建数据库管理器实例 +# connection = "postgresql+psycopg2://postgres:12345678@localhost:5432/agent_controller" +# connection = "sqlite:///Agent/src/agent/history/data_base/chat_history.db" +# _db_manager = PostgreSQL_Manager(connection) + +# # 创建工具实例 +# _all_database_tools = [ +# Websearchclass(), +# DateQueryClass(), +# ListTableComment(db_manager=_db_manager), +# ListTableSchema(db_manager=_db_manager), +# SQLQuery(db_manager=_db_manager), +# SQLQueryChecker(db_manager=_db_manager) +# ] +# def get_sql_tools(): +# return _all_database_tools + + +_all_tools = [ + Websearchclass(), + DateQueryClass(), + ScreenCaptureDescribeClass(), + BrowserControlClass() +] + +def get_tools(): + return _all_tools diff --git a/src/agent/tools/__pycache__/__init__.cpython-312.pyc b/src/agent/tools/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..3950798 Binary files /dev/null and b/src/agent/tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/tool.cpython-312.pyc b/src/agent/tools/__pycache__/tool.cpython-312.pyc new file mode 100644 index 0000000..e2dbc5e Binary files /dev/null and b/src/agent/tools/__pycache__/tool.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/tool_main.cpython-312.pyc b/src/agent/tools/__pycache__/tool_main.cpython-312.pyc new file mode 100644 index 0000000..e992f31 Binary files /dev/null and b/src/agent/tools/__pycache__/tool_main.cpython-312.pyc differ diff --git a/src/agent/tools/tool.py b/src/agent/tools/tool.py new file mode 100644 index 0000000..af820a0 --- /dev/null +++ b/src/agent/tools/tool.py @@ -0,0 +1,54 @@ +from langchain.tools import tool + +@tool("web_search") +def web_search(query: str) -> str: + """ + 一个用于执行网络搜索的工具。它接收一个查询语句作为输入，并返回 Google 的搜索结果。 + + Args: + query (str): 搜索查询内容 + + Returns: + str: 格式化后的搜索结果 + """ + try: + from langchain_community.utilities import GoogleSerperAPIWrapper + # 这里直接使用了一个示例的 Serper API 密钥，实际使用时请替换为你自己的密钥 + SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5" + # LangChain 官方封装的 Serper API 工具 + search = GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048) + # 直接调用 run 方法获取结果，LangChain 已处理好格式 + return search.run(query) + except Exception as e: + return f"执行网络搜索时出错: {str(e)}" + +@tool("date_query") +def date_query() -> str: + """ + 一个用于执行日期查询的工具。它接收一个查询语句作为输入，并返回相应的日期结果。 + + Returns: + str: 格式化后的日期查询结果 + """ + try: + # 显示当前日期和时间 + from datetime import datetime + now = datetime.now() + return now.strftime("%Y-%m-%d %H:%M:%S") + + except Exception as e: + return f"执行日期查询时出错: {str(e)}" + +if __name__ == "__main__": + # print(web_search.name) # 输出工具名称 + # print(web_search.description) # 输出工具描述 + # print(web_search.args_schema.model_json_schema()) # 输出工具参数结构 + # # 测试工具 + # query = "Python 中的列表和元组有什么区别？" + # result = web_search.invoke({"query": query}) + # print(result) + print(date_query.name) # 输出工具名称 + print(date_query.description) # 输出工具描述 + print(date_query.args_schema.model_json_schema()) # 输出工具参数结构 + print(date_query.invoke({})) # 测试工具 + diff --git a/src/agent/tools/tool_database_class.py b/src/agent/tools/tool_database_class.py new file mode 100644 index 0000000..947c95f --- /dev/null +++ b/src/agent/tools/tool_database_class.py @@ -0,0 +1,118 @@ +from langchain.tools import BaseTool +from pydantic import Field, create_model, BaseModel +from utils.database_utils import PostgreSQL_Manager +import json +from typing import Optional + +class ListTableComment(BaseTool): + + name: str = "list_table_comment" + description: str = "一个列出数据库中所有表名和对应描述的工具，可以根据用户的查询返回相关的表名和描述列表。" + db_manager : PostgreSQL_Manager + + def _run(self, query: str) -> str: + # 这里可以实现具体的列出数据库表的逻辑，例如调用数据库管理类的方法 + # 下面是一个示例实现，实际使用时请替换为你自己的数据库查询逻辑 + try: + result_json = self.db_manager.get_table_comments() + results = json.loads(result_json) + if not isinstance(results, dict): + raise ValueError("get_table_comments did not return a JSON object") + result = f"数据库中共有 {len(results)} 张表" + for i, (table_name, comment) in enumerate(results.items()): + if not comment or comment.isspace(): + comment = "(暂无描述)" + result += f"\n{i+1}. {table_name}: {comment}" + return result + except Exception as e: + return f"执行列出数据库表和描述时出错: {str(e)}" + + async def _arun(self, query: str) -> str: + return self._run(query) + +class ListTableSchema(BaseTool): + + name: str = "list_table_schema" + description: str = "一个列出数据库中表名对应模式的工具，可以根据用户的查询返回相关的表名和模式信息，包括列定义、主键、外键等。输入应为逗号分隔的表名列表。" + db_manager: PostgreSQL_Manager + args_schema: type[BaseModel] = create_model("Input_type", table_names=(Optional[str], Field(description="要查询的表名列表,如果为None则返回所有表的模式信息,例如:'table1', 'table2'"))) + + def _run(self, table_names : Optional[str] = None) -> str: + try: + if table_names: + table_list = [name.strip() for name in table_names.split(",")] + result_json = self.db_manager.get_table_schema(table_list) + results = json.loads(result_json) + if not isinstance(results, dict): + raise ValueError("get_table_schema did not return a JSON object") + return json.dumps(results, ensure_ascii=False, indent=2) + except Exception as e: + return f"执行列出数据库表和模式时出错: {str(e)}" + + async def _arun(self, table_names : Optional[str] = None) -> str: + return self._run(table_names) + +class SQLQuery(BaseTool): + + name: str = "sql_query" + description: str = "在PostgreSQL数据库中执行安全的SELECT或WITH查询并返回结果。" + db_manager: PostgreSQL_Manager + args_schema: type[BaseModel] = create_model("Input_type", query=(str, Field(description="要执行的SQL SELECT查询语句"))) + + def _run(self, query: str) -> str: + try: + result_json = self.db_manager.execute_query(query) + results = json.loads(result_json) + if isinstance(results, list): + # 成功返回数据行 + return json.dumps(results, ensure_ascii=False, indent=2) + elif isinstance(results, dict): + if "error" in results: + # 返回错误信息 + return f"查询执行失败: {results['error']}" + elif "message" in results: + # 查询成功但无数据返回 + return results["message"] + else: + # 其他字典格式的结果 + return json.dumps(results, ensure_ascii=False, indent=2) + else: + raise ValueError(f"execute_query 返回了意外的数据类型: {type(results)}") + except Exception as e: + return f"执行SQL查询时出错: {str(e)}" + + async def _arun(self, query: str) -> str: + return self._run(query) + +class SQLQueryChecker(BaseTool): + + name: str = "sql_query_checker" + description: str = "一个检查SQL查询语句是否安全的工具，主要用于防止SQL注入攻击。输入为SQL查询语句，输出为检查结果。" + args_schema: type[BaseModel] = create_model("Input_type", query=(str, Field(description="要检查的SQL查询语句"))) + db_manager: PostgreSQL_Manager + + def _run(self, query: str) -> str: + try: + result_json = self.db_manager.validate_query(query) + result = json.loads(result_json) + if not isinstance(result, dict): + raise ValueError("validate_query did not return a JSON object") + return json.dumps(result, ensure_ascii=False, indent=2) + except Exception as e: + return f"执行SQL查询安全检查时出错: {str(e)}" + async def _arun(self, query: str) -> str: + return self._run(query) + +if __name__ == "__main__": + manager = PostgreSQL_Manager("postgresql+psycopg2://postgres:12345678@localhost:5432/agent_controller") + list_table_tool = ListTableComment(db_manager=manager) + print(list_table_tool.run("列出数据库中的所有表和描述")) + # list_table_tool = ListTableSchema(db_manager=manager) + # print(list_table_tool.run({"table_names": []})) + # sql_query = SQLQuery(db_manager=manager) + # print(sql_query.run("SELECT * FROM tasks;")) + # sql_query_checker = SQLQueryChecker(db_manager=manager) + # print("Valid query:") + # print(sql_query_checker.run("SELECT * FROM tasks;")) + # print("\nInvalid query:") + # print(sql_query_checker.run("DROP TABLE tasks;")) diff --git a/src/agent/tools/tool_main.py b/src/agent/tools/tool_main.py new file mode 100644 index 0000000..ff2e9f5 --- /dev/null +++ b/src/agent/tools/tool_main.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import io +from typing import Optional, Literal +import sys +import time +import webbrowser +import pyautogui + +from langchain.tools import BaseTool +from pydantic import BaseModel, Field, create_model + +from agent.llm import Qwen36_35B_A3B_FP8_LLM +from utils.env_utils import BROWSER_MAP + + +class Websearchclass(BaseTool): + + args_schema : type[BaseModel] = create_model("Input_type", query=(str, Field(description="搜索查询内容"))) + name : str = "web_search_class" + description : str = "一个网络搜索工具，可以根据用户的查询返回相关的搜索结果。仅当需要查询近期知识或已有知识无法解答需要联网查询时调用，其余任何情况严禁调用。" + + def _get_search_wrapper(self): + from langchain_community.utilities import GoogleSerperAPIWrapper + SERPER_API_KEY = "d6348dcce0c72bb8102f0eacfd2f536f797da0c5" + return GoogleSerperAPIWrapper(serper_api_key=SERPER_API_KEY, content_size=2048) + + def _run(self, query: str) -> str: + try: + return self._get_search_wrapper().run(query) + except Exception as e: + return f"执行网络搜索时出错: {str(e)}" + + async def _arun(self, query: str) -> str: + try: + return await self._get_search_wrapper().arun(query) + except Exception as e: + return f"执行网络搜索时出错: {str(e)}" + +class DateQueryClass(BaseTool): + + # 定义工具的输入参数结构，这里没有输入参数，所以是一个空的模型 + args_schema : type[BaseModel] = create_model("NoneInput") + name : str = "date_query_class" + description : str = "一个日期查询工具，可以返回当前的日期信息。仅当需要查询当前时间时调用，其余任何情况严禁调用。" + + def _run(self) -> str: + try: + from datetime import datetime + now = datetime.now() + return now.strftime("%Y-%m-%d %H:%M:%S") + except Exception as e: + return f"执行日期查询时出错: {str(e)}" + + async def _arun(self) -> str: + return self._run() + + +class ScreenCaptureDescribeClass(BaseTool): + """截取当前屏幕并用与对话上传图片相同的视觉链路解析为文字，供 Agent 推理。""" + + args_schema: type[BaseModel] = create_model( + "ScreenCaptureInput", + focus_query=( + Optional[str], + Field( + default=None, + description="可选。希望重点观察的界面或问题，例如「错误提示全文」「浏览器地址栏」；留空则整体描述屏幕。", + ), + ), + ) + name: str = "screen_capture_describe" + description: str = ( + "截取当前主屏幕画面，并用视觉模型转为文字描述。" + "注意：这个工具仅用于获取当前屏幕的实时画面，其余任何情况严禁调用。" + ) + + def _run(self, focus_query: Optional[str] = None) -> str: + # 1. 确保 pyautogui 可用 + try: + import pyautogui + except ImportError as e: + return ( + f"未安装 pyautogui，无法截屏: {e}。" + "请在项目环境中执行: pip install pyautogui" + ) + + # 2. 截屏并转为 PNG 字节流 + try: + pil_img = pyautogui.screenshot() + buf = io.BytesIO() + pil_img.save(buf, format="PNG") + raw_png = buf.getvalue() + except Exception as e: + return f"截屏失败: {e}" + + # 3. 构建描述指令 + query = (focus_query or "").strip() + + # 4. 调用与对话图片上传完全相同的视觉模型进行描述 + try: + # Qwen36_35B_A3B_FP8_LLM 为全局 ChatQwen 实例，已在模块中定义 + text = Qwen36_35B_A3B_FP8_LLM.describe_image_bytes(raw_png, prompt=query) + except Exception as e: + return f"屏幕图像视觉解析失败: {e}" + + if not (text or "").strip(): + return "（已截屏，但模型未返回有效描述。）" + return f"【当前屏幕视觉描述】\n{text.strip()}" + +class BrowserControlClass(BaseTool): + """统一浏览器控制工具：支持打开/关闭浏览器、刷新/关闭标签页、打开指定 URL""" + + args_schema: type[BaseModel] = create_model( + "BrowserControlInput", + action=(Literal["open", "close"], Field( + default="open", + description="操作类型：'open' 打开/刷新，'close' 关闭" + )), + url=(Optional[str], Field( + default=None, + description="URL 地址、'current' 表示当前页，留空则操作整个浏览器窗口" + )), + browser=(Optional[str], Field( + default=None, + description="浏览器名称，如 'edge', 'chrome', 'firefox'，不指定则使用系统默认" + )), + ) + name: str = "browser_control" + description: str = ( + "统一控制浏览器：打开/关闭浏览器、刷新/关闭标签页、打开指定 URL。\n" + "- action='open', url=None: 打开浏览器首页\n" + "- action='open', url='current': 刷新当前标签页\n" + "- action='open', url='https://...': 在新标签页打开指定 URL\n" + "- action='close', url=None: 关闭整个浏览器窗口\n" + "- action='close', url='current': 关闭当前标签页\n" + "- action='close', url='https://...': 关闭当前活动标签页" + "仅用于进行浏览器操作时调用，其余任何情况严禁调用。" + ) + + # ---------- 辅助方法 ---------- + def _get_browser_info(self, browser_name: Optional[str]): + """获取浏览器控制器和窗口标题关键字""" + if browser_name is None: + return webbrowser.get(), None + + name_lower = browser_name.lower().strip() + if name_lower not in BROWSER_MAP: + raise ValueError(f"不支持的浏览器: {browser_name}，支持: {', '.join(BROWSER_MAP.keys())}") + + reg_name, title_keyword = BROWSER_MAP[name_lower] + try: + controller = webbrowser.get(reg_name) + except webbrowser.Error: + controller = webbrowser.get(browser_name) + return controller, title_keyword + + def _activate_window(self, title_keyword: str) -> bool: + """激活浏览器窗口（仅 Windows）""" + if sys.platform != "win32": + return False + try: + time.sleep(0.3) + windows = pyautogui.getWindowsWithTitle(title_keyword) + if windows: + win = windows[0] + if win.isMinimized: + win.restore() + win.activate() + return True + except Exception: + pass + return False + + def _send_keys(self, *keys): + """发送快捷键，根据平台调整""" + pyautogui.hotkey(*keys) + + # ---------- 核心操作 ---------- + def _open_action(self, url: Optional[str], browser: Optional[str]): + """执行打开类操作""" + controller, title_keyword = self._get_browser_info(browser) + + if url is None: + # 仅打开浏览器首页（激活或启动） + controller.open("about:blank") + if title_keyword: + self._activate_window(title_keyword) + browser_info = f" (使用 {browser})" if browser else "" + return f"已打开浏览器首页{browser_info}" + + elif url.lower() == "current": + # 刷新当前页 + if title_keyword: + if not self._activate_window(title_keyword): + return f"未找到 {browser or '默认'} 浏览器窗口，无法刷新" + time.sleep(0.2) + if sys.platform == "darwin": + self._send_keys("command", "r") + else: + self._send_keys("f5") + return "已刷新当前标签页" + + else: + # 打开指定 URL + controller.open(url) + if title_keyword: + self._activate_window(title_keyword) + return f"已尝试打开 URL: {url}" + + def _close_action(self, url: Optional[str], browser: Optional[str]): + """执行关闭类操作""" + _, title_keyword = self._get_browser_info(browser) + + if url is None: + # 关闭整个浏览器 + if title_keyword: + if not self._activate_window(title_keyword): + return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭" + time.sleep(0.2) + if sys.platform == "darwin": + self._send_keys("command", "q") + else: + self._send_keys("alt", "f4") + return "已尝试关闭浏览器窗口" + + elif url.lower() == "current": + # 关闭当前标签页 + if title_keyword: + if not self._activate_window(title_keyword): + return f"未找到 {browser or '默认'} 浏览器窗口，无法关闭标签页" + time.sleep(0.2) + if sys.platform == "darwin": + self._send_keys("command", "w") + else: + self._send_keys("ctrl", "w") + return "已尝试关闭当前标签页" + + def _run(self, action: str = "open", url: Optional[str] = None, browser: Optional[str] = None) -> str: + try: + if action == "open": + return self._open_action(url, browser) + elif action == "close": + return self._close_action(url, browser) + else: + return f"不支持的操作类型: {action}" + except Exception as e: + return f"执行浏览器操作时出错: {e}" + +if __name__ == "__main__": + # 这里可以进行一些简单的测试，例如创建工具实例并调用方法 + pass diff --git a/src/agent/utils/__init__.py b/src/agent/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/agent/utils/__pycache__/__init__.cpython-312.pyc b/src/agent/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..c72cf9f Binary files /dev/null and b/src/agent/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc b/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc new file mode 100644 index 0000000..37b3922 Binary files /dev/null and b/src/agent/utils/__pycache__/agent_utils.cpython-312.pyc differ diff --git a/src/agent/utils/__pycache__/env_utils.cpython-312.pyc b/src/agent/utils/__pycache__/env_utils.cpython-312.pyc new file mode 100644 index 0000000..e8a9f47 Binary files /dev/null and b/src/agent/utils/__pycache__/env_utils.cpython-312.pyc differ diff --git a/src/agent/utils/__pycache__/image_utils.cpython-312.pyc b/src/agent/utils/__pycache__/image_utils.cpython-312.pyc new file mode 100644 index 0000000..8894438 Binary files /dev/null and b/src/agent/utils/__pycache__/image_utils.cpython-312.pyc differ diff --git a/src/agent/utils/agent_utils.py b/src/agent/utils/agent_utils.py new file mode 100644 index 0000000..f8e77ab --- /dev/null +++ b/src/agent/utils/agent_utils.py @@ -0,0 +1,181 @@ +# agent/agent_utils.py +import json +import os +import shutil + +from langchain_community.chat_message_histories import SQLChatMessageHistory +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder + +from agent.memory.system_prompt import SUMMARY_SYSTEM_PROMPT + +# 关键改动：__file__ 在 agent/utils/agent_utils.py 下， +# 取两次 dirname 得到 agent 目录（即 src/agent） +_AGENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +HISTORY_ROOT = os.path.join(_AGENT_DIR, "history") +_DB_DIR = os.path.join(HISTORY_ROOT, "data_base") +CHAT_HISTORY_DB_FS = os.path.join(_DB_DIR, "chat_history.db") +MULTI_INPUTS_ROOT = os.path.join(HISTORY_ROOT, "multi_inputs") + +def ensure_dirs_and_migrate_db() -> None: + os.makedirs(_DB_DIR, exist_ok=True) + os.makedirs(os.path.join(HISTORY_ROOT, "multi_inputs"), exist_ok=True) + old_db = os.path.join(HISTORY_ROOT, "chat_history.db") + if os.path.isfile(old_db) and not os.path.isfile(CHAT_HISTORY_DB_FS): + shutil.move(old_db, CHAT_HISTORY_DB_FS) + +ensure_dirs_and_migrate_db() + +def get_history(session_id: str) -> SQLChatMessageHistory: + return SQLChatMessageHistory( + session_id=session_id, + connection="sqlite:///" + CHAT_HISTORY_DB_FS.replace("\\", "/"), + ) + +def summarize_history(query, config): + """ + 剪辑最近8条消息，并对更早的消息生成摘要。 + 返回 {'summary': str|None, 'chat_history': list[BaseMessage]} + """ + session_id = config["configurable"]["session_id"] + if not session_id: + raise ValueError("Session ID is required in the config.") + chat_history = get_history(session_id) + stored_messages = chat_history.messages + + if len(stored_messages) <= 8: + return {"summary": None, "chat_history": stored_messages} + + recent = stored_messages[-8:] + old = stored_messages[:-8] + + summary_prompt = ChatPromptTemplate.from_messages([ + ("system", SUMMARY_SYSTEM_PROMPT), + MessagesPlaceholder(variable_name="messages_to_summarize", optional=True), + ("human", "请生成包含上述对话核心内容的摘要，保留重要事实和决策。"), + ]) + + llm = config.get("llm") + chain = summary_prompt | llm + summary = chain.invoke( + {"messages_to_summarize": old}, + config={ + "metadata": { + "output_type": "text", + "audio_output_dir": None, + } + } + ) + return {"summary": summary.content, "chat_history": recent} + + +def assistant_text(value) -> str: + """把网关返回的 content、reasoning 等（str 或块列表）收成一段可见文本。""" + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts: list[str] = [] + for item in value: + if isinstance(item, str) and item.strip(): + parts.append(item.strip()) + elif isinstance(item, dict): + for key in ("text", "reasoning_content", "content"): + v = item.get(key) + if isinstance(v, str) and v.strip(): + parts.append(v.strip()) + break + return "\n".join(parts).strip() + if isinstance(value, dict): + for key in ("text", "content", "input", "output", "query", "answer", "result"): + v = value.get(key) + if isinstance(v, str) and v.strip(): + return v.strip() + try: + return json.dumps(value, ensure_ascii=False) + except TypeError: + return str(value).strip() + return str(value).strip() + + +def normalize_user_text_content(value) -> str: + """尽量把用户输入收敛为纯文本，避免把 dict/list 结构直接送进主模型。""" + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, dict): + for key in ("text", "content", "input", "query", "question", "message"): + v = value.get(key) + if isinstance(v, str) and v.strip(): + return v.strip() + content = value.get("content") + if content is not None: + text = normalize_user_text_content(content) + if text: + return text + return assistant_text(value) + if isinstance(value, list): + parts: list[str] = [] + for item in value: + if isinstance(item, str) and item.strip(): + parts.append(item.strip()) + elif isinstance(item, dict): + if item.get("type") == "text": + text = (item.get("text") or "").strip() + if text: + parts.append(text) + else: + text = normalize_user_text_content(item) + if text: + parts.append(text) + return "\n".join(parts).strip() + return str(value).strip() + + +def extract_result_from_invoke(value) -> str: + """兼容不同 invoke 返回结构，尽量提取可显示的助手文本。""" + if value is None: + return "" + + if hasattr(value, "content"): + text = assistant_text(getattr(value, "content", None)) + if text: + return text + + if isinstance(value, dict): + for key in ("output", "content", "text", "answer", "result"): + text = assistant_text(value.get(key)) + if text: + return text + + message = value.get("message") + if message is not None: + text = extract_result_from_invoke(message) + if text: + return text + + generations = value.get("generations") + if isinstance(generations, list): + for generation in generations: + text = extract_result_from_invoke(generation) + if text: + return text + + if isinstance(value, list): + for item in value: + text = extract_result_from_invoke(item) + if text: + return text + + if hasattr(value, "message"): + text = extract_result_from_invoke(getattr(value, "message", None)) + if text: + return text + + if hasattr(value, "additional_kwargs"): + text = assistant_text(getattr(value, "additional_kwargs", {}).get("reasoning_content")) + if text: + return text + + return "" \ No newline at end of file diff --git a/src/agent/utils/database_utils.py b/src/agent/utils/database_utils.py new file mode 100644 index 0000000..6b45d36 --- /dev/null +++ b/src/agent/utils/database_utils.py @@ -0,0 +1,255 @@ +from sqlalchemy import create_engine, inspect, text +import json +from typing import Optional, List +from uuid import UUID +from datetime import datetime + +class JSONEncoder(json.JSONEncoder): + """自定义JSON编码器，处理非标准序列化类型""" + def default(self, obj): + if isinstance(obj, UUID): + return str(obj) + elif isinstance(obj, datetime): + return obj.isoformat() + elif hasattr(obj, '__dict__'): + return str(obj) + return super().default(obj) + +def make_json_serializable(obj): + """递归转换对象为JSON序列化格式""" + if isinstance(obj, dict): + return {k: make_json_serializable(v) for k, v in obj.items()} + elif isinstance(obj, (list, tuple)): + return [make_json_serializable(item) for item in obj] + elif isinstance(obj, UUID): + return str(obj) + elif isinstance(obj, datetime): + return obj.isoformat() + elif hasattr(obj, '__dict__'): + return str(obj) + else: + return obj + +class PostgreSQL_Manager: + def __init__(self, connection_params): + """ + PostgreSQL数据库管理类,提供连接和操作数据库的功能 + Args: + connection_params (dict): 连接参数字符串，格式为： + postgresql+psycopg2://用户名:密码@主机:端口/数据库名 + 本机默认为:postgresql+psycopg2://postgres:12345678@localhost:5432/agent_controller + """ + self.engine = create_engine(connection_params, pool_size=5, pool_recycle=7200) + + def get_table_names(self) -> list[str]: + """ + 获取数据库中的所有表名 + Returns: + list: 数据库中的表名列表 + """ + try: + # 使用SQLAlchemy的inspect功能获取数据库表名 + inspector = inspect(self.engine) + return inspector.get_table_names() + except Exception as e: + print(f"Error occurred while fetching table names: {e}") + return [] + + def get_table_comments(self, table_name: str = None) -> str: + """ + 获取指定表/所有表的注释信息,并整理成JSON格式 + Args: + table_name (str, optional): 表名,如果为None则返回所有表 + Returns: + str: JSON格式的表的注释信息 + """ + try: + inspector = inspect(self.engine) + if table_name is None: # 返回所有表的注释 + table_names = inspector.get_table_names() + comments_dict = {table: inspector.get_table_comment(table).get('text', '') for table in table_names} + serializable_dict = make_json_serializable(comments_dict) + return json.dumps(serializable_dict, ensure_ascii=False, indent=2, cls=JSONEncoder) + else: + comment = inspector.get_table_comment(table_name).get('text', '') + result = {"table_name": table_name, "comment": comment} + serializable_result = make_json_serializable(result) + return json.dumps(serializable_result, ensure_ascii=False, indent=2, cls=JSONEncoder) + except Exception as e: + error_msg = {"error": f"Error occurred while fetching table comments: {str(e)}"} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + + def get_table_schema(self, table_names: Optional[List[str]] = None) -> str: + """ + 获取指定表/所有表的schema信息,并整理成JSON格式 + Args: + table_names (List[str], optional): 表名列表,如果为None则返回所有表 + Returns: + str: JSON格式的表的schema信息 + """ + try: + inspector = inspect(self.engine) + table_schema = {} + names_to_query = table_names if table_names else inspector.get_table_names() + for table_name in names_to_query: + columns = inspector.get_columns(table_name) + pk_constraints = inspector.get_pk_constraint(table_name) + primary_keys = pk_constraints['constrained_columns'] if pk_constraints else [] + foreign_keys = inspector.get_foreign_keys(table_name) + indexes = inspector.get_indexes(table_name) + table_schema[table_name] = { + 'columns': columns, + 'primary_keys': primary_keys, + 'foreign_keys': foreign_keys, + 'indexes': indexes + } + # 转换为JSON序列化格式 + serializable_schema = make_json_serializable(table_schema) + return json.dumps(serializable_schema, ensure_ascii=False, indent=2, cls=JSONEncoder) + except Exception as e: + error_msg = {"error": f"Error occurred while fetching table schema: {str(e)}"} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + + def execute_query(self, query: str) -> str: + """ + 执行任意SQL查询,并返回结果 + Args: + query (str): SQL查询语句 + Returns: + str: JSON格式的查询结果或错误信息 + """ + # 安全检查，防止执行危险的SQL语句，仅允许select和with查询语句 + if not query.strip().lower().startswith(('select', 'with')): + error_msg = {"error": "Only SELECT and WITH queries are allowed."} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + + try: + with self.engine.connect() as conn: + result = conn.execute(text(query)) + # 获取列名 + columns = result.keys() + # 获取数据（限制返回数据量防止内存溢出） + rows = result.fetchmany(100) # 只获取前100行数据 + if not rows: + return json.dumps({"message": "Query executed successfully, no rows returned."}, ensure_ascii=False, indent=2) + else: + result_data = [] + for row in rows: + row_dict = dict(zip(columns, row)) + result_data.append(row_dict) + # 转换为JSON序列化格式 + serializable_data = make_json_serializable(result_data) + return json.dumps(serializable_data, ensure_ascii=False, indent=2, cls=JSONEncoder) + except Exception as e: + error_msg = {"error": f"Error occurred while executing query: {str(e)}"} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + + def validate_query(self, query: str) -> str: + """ + 验证SQL查询语句的合法性 + Args: + query (str): SQL查询语句 + Returns: + bool: 查询语句是否合法 + """ + if not query or not query.strip(): + error_msg = {"error": f"Error occurred while validating query: Query is must not be empty."} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + + # 简单的安全检查，防止执行危险的SQL语句，仅允许select和with查询语句 + if not query.strip().lower().startswith(('select', 'with')): + error_msg = {"error": f"Error occurred while validating query: Only SELECT and WITH queries are allowed."} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + + try: + with self.engine.connect() as conn: + if self.engine.dialect.name == 'postgresql': + # 使用EXPLAIN分析查询计划来验证查询的合法性 + explain_query = f"EXPLAIN {query}" + else: + # 对于其他数据库，可以直接执行查询但不获取结果来验证合法性 + explain_query = query + conn.execute(text(explain_query)) + return json.dumps({"message": "Query is valid."}, ensure_ascii=False, indent=2) + except Exception as e: + error_msg = {"error": f"Error occurred while validating query: {str(e)}"} + return json.dumps(error_msg, ensure_ascii=False, indent=2) + # def rename_table(self, old_table_name: str, new_table_name: str) -> bool: + # """ + # 重命名数据库表 + # Args: + # old_table_name (str): 原表名 + # new_table_name (str): 新表名 + # Returns: + # bool: 操作是否成功 + # """ + # try: + # with self.engine.connect() as conn: + # sql = text(f"ALTER TABLE {old_table_name} RENAME TO {new_table_name}") + # conn.execute(sql) + # conn.commit() + # print(f"Table '{old_table_name}' renamed to '{new_table_name}' successfully.") + # return True + # except Exception as e: + # print(f"Error occurred while renaming table: {e}") + # return False + + # def update_table_comment(self, table_name: str, comment: str) -> bool: + # """ + # 更新表的注释 + # Args: + # table_name (str): 表名 + # comment (str): 新的注释内容 + # Returns: + # bool: 操作是否成功 + # """ + # try: + # with self.engine.connect() as conn: + # sql = text(f"COMMENT ON TABLE {table_name} IS '{comment}'") + # conn.execute(sql) + # conn.commit() + # print(f"Comment for table '{table_name}' updated successfully.") + # return True + # except Exception as e: + # print(f"Error occurred while updating table comment: {e}") + # return False +if __name__ == "__main__": + DB_CONFIG = { + "host": "localhost", + "port": 5432, + "database": "agent_controller", + "user": "postgres", + "password": "12345678" + } + connection_params = f"postgresql+psycopg2://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['database']}?client_encoding=utf8" + db_manager = PostgreSQL_Manager(connection_params) + + # 获取并打印所有表名和注释（JSON格式） + print("All table comments (JSON):") + all_comments_json = db_manager.get_table_comments() + print(all_comments_json) + + # 示例：获取特定表的注释（JSON格式） + # table_names = db_manager.get_table_names() + # if table_names: + # print(f"\nSpecific table comment for '{table_names[0]}' (JSON):") + # specific_comment_json = db_manager.get_table_comments(table_names[0]) + # print(specific_comment_json) + + # table_name = "result_artifacts" + # new_comment = "用于存放任务运行结果的表" + # db_manager.update_table_comment(table_name, new_comment) + + # table_name = "agent_state_history" + # new_comment = "用于存放agent状态历史的表" + # db_manager.update_table_comment(table_name, new_comment) + + # table_name = "agents" + # new_comment = "用于存放agent的注册表" + # db_manager.update_table_comment(table_name, new_comment) + + # table_name = "tasks" + # new_comment = "用于存放任务的仓库表" + # db_manager.update_table_comment(table_name, new_comment) + + \ No newline at end of file diff --git a/src/agent/utils/env_utils.py b/src/agent/utils/env_utils.py new file mode 100644 index 0000000..7ccca83 --- /dev/null +++ b/src/agent/utils/env_utils.py @@ -0,0 +1,36 @@ +import os +from typing import Dict, Tuple + +from dotenv import load_dotenv +load_dotenv(override=True) + + +def _get_env(name: str, default: str | None = None, cast: type | None = None, required: bool = False): + value = os.getenv(name, default) + if value is None and required: + raise EnvironmentError(f"Missing required environment variable: {name}") + if cast is None or value is None: + return value + try: + return cast(value) + except (ValueError, TypeError) as exc: + raise ValueError(f"Invalid value for {name}: {value}") from exc + + +LLM_BASE_URL = _get_env("LLM_BASE_URL", required=True) +LLM_API_KEY = _get_env("LLM_API_KEY", required=True) +LLM_MODEL = _get_env("LLM_MODEL", required=True) +LLM_TIMEOUT = _get_env("LLM_TIMEOUT", default="60", cast=float) +LLM_REQUESTS_PER_SECOND = _get_env("LLM_REQUESTS_PER_SECOND", default="10", cast=float) +LLM_MAX_BUCKET_SIZE = _get_env("LLM_MAX_BUCKET_SIZE", default="10", cast=int) +LLM_CHECK_EVERY_N_SECONDS = _get_env("LLM_CHECK_EVERY_N_SECONDS", default="0.1", cast=float) +ASR_TTS_BASE_URL = _get_env("ASR_TTS_BASE_URL") + +BROWSER_MAP: Dict[str, Tuple[str, str]] = { + "edge": ("microsoft-edge", "Microsoft Edge"), + "chrome": ("chrome", "Google Chrome"), + "firefox": ("firefox", "Mozilla Firefox"), + "safari": ("safari", "Safari"), + "opera": ("opera", "Opera"), + "brave": ("brave", "brave browser"), +} diff --git a/src/agent/utils/image_utils.py b/src/agent/utils/image_utils.py new file mode 100644 index 0000000..58282c8 --- /dev/null +++ b/src/agent/utils/image_utils.py @@ -0,0 +1,143 @@ +"""图片缩放、体积控制与 OpenAI 风格多模态 image_url 片段（供 LLM / Agent 使用）。""" +from __future__ import annotations + +import base64 +import io +import os +from typing import Any, Protocol + +from PIL import Image + + +class SupportsChatCompletion(Protocol): + """具备 ``chat_completion(messages, **kwargs)`` 的 LLM（如 ``ChatQwen35OpenAI``）。""" + + def chat_completion(self, messages: list[Any], **kwargs: Any) -> Any: ... + +# 小写扩展名（含点），用于判断路径是否为支持的栅格图 +IMAGE_FILE_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp"}) + + +def guess_image_mime(extension_without_dot: str) -> str: + """根据扩展名返回 image/* MIME，未知则回退为 image/jpeg。""" + ext = extension_without_dot.lower().lstrip(".") + if ext in ("png", "jpg", "jpeg", "gif", "bmp", "webp"): + return "image/jpeg" if ext == "jpg" else f"image/{ext}" + return "image/jpeg" + + +def resize_image_if_needed(image_path_or_bytes: str | bytes, *, max_size: int = 768) -> bytes: + """ + 控制图片体积，避免 data-url base64 触发超长 token。 + - 若图片尺寸过大，等比例缩放（默认最大边 max_size） + - 优先压成 JPEG（透明图保底转白底 RGB） + """ + if isinstance(image_path_or_bytes, str): + img = Image.open(image_path_or_bytes) + original_format = img.format or "PNG" + else: + img = Image.open(io.BytesIO(image_path_or_bytes)) + original_format = img.format or "PNG" + + if img.mode == "RGBA": + background = Image.new("RGB", img.size, (255, 255, 255)) + background.paste(img, mask=img.split()[-1]) + img = background + original_format = "JPEG" + elif img.mode == "LA": + background = Image.new("L", img.size, 255) + background.paste(img, mask=img.split()[-1]) + img = background + + original_width, original_height = img.size + + if original_width > max_size or original_height > max_size: + ratio = min(max_size / original_width, max_size / original_height) + new_width = int(original_width * ratio) + new_height = int(original_height * ratio) + img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) + print(f"图片已从 {original_width}x{original_height} 缩放到 {new_width}x{new_height}") + else: + print(f"图片尺寸 {original_width}x{original_height} 未超过限制，无需缩放") + + img_buffer = io.BytesIO() + save_format = (original_format or "PNG").upper() + if save_format not in ("PNG", "JPEG", "JPG"): + save_format = "JPEG" + + if save_format in ("JPEG", "JPG"): + if img.mode not in ("RGB", "L"): + img = img.convert("RGB") + img.save( + img_buffer, + format="JPEG", + quality=70, + optimize=True, + progressive=True, + ) + else: + img.save(img_buffer, format="PNG", optimize=True) + img_buffer.seek(0) + return img_buffer.getvalue() + + +def image_to_base64(image_content: str | bytes) -> str | None: + """将图片路径或原始字节转为 base64 字符串；失败返回 None。""" + try: + processed = resize_image_if_needed(image_content) + return base64.b64encode(processed).decode("utf-8") + except Exception as e: + print(f"处理图片 {image_content!r} 时出错: {e}") + return None + + +def _mime_from_magic(data: bytes) -> str: + """根据压缩后图片文件头推断 MIME，用于内存字节构造 data URL。""" + if len(data) >= 2 and data[:2] == b"\xff\xd8": + return "image/jpeg" + if len(data) >= 8 and data[:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if len(data) >= 6 and data[:6] in (b"GIF87a", b"GIF89a"): + return "image/gif" + return "image/jpeg" + + +def image_bytes_to_openai_image_url_part( + image_bytes: bytes, + *, + detail: str | None = None, +) -> dict | None: + """将内存中的图片字节（如截屏 PNG）转为 OpenAI 风格的 image_url 块；失败返回 None。""" + try: + processed = resize_image_if_needed(image_bytes) + except Exception as e: + print(f"处理图片字节时出错: {e}") + return None + mime = _mime_from_magic(processed) + b64_str = base64.b64encode(processed).decode("utf-8") + url = f"data:{mime};base64,{b64_str}" + image_url: dict = {"url": url} + if detail: + image_url["detail"] = detail + return {"type": "image_url", "image_url": image_url} + + +def image_path_to_openai_image_url_part( + file_path: str, + *, + detail: str | None = None, +) -> dict | None: + """ + 从本地图片路径构造 OpenAI Chat Completions 的 image_url 内容块。 + 无法编码时返回 None。 + """ + b64_str = image_to_base64(file_path) + if not b64_str: + return None + ext = os.path.splitext(file_path)[1].lower().lstrip(".") + mime_type = guess_image_mime(ext) + url = f"data:{mime_type};base64,{b64_str}" + image_url: dict = {"url": url} + if detail: + image_url["detail"] = detail + return {"type": "image_url", "image_url": image_url} diff --git a/src/agent/utils/log_utils.py b/src/agent/utils/log_utils.py new file mode 100644 index 0000000..b36cb9e --- /dev/null +++ b/src/agent/utils/log_utils.py @@ -0,0 +1,38 @@ +import sys +from loguru import logger + +# Trace < Debug < Info < Success < Warning < Error < Critical + + +class MyLogger: + def __init__(self): + self.logger = logger + self.logger.remove() + self.logger.add( + sys.stdout, + level="DEBUG", + format="{time:YYYYMMDD HH:mm:ss} | " + "{process.name} | " + "{thread.name} | " + "{module}.{function}" + ":{line} | " + "{level}: " + "{message}", + ) + + def get_logger(self): + return self.logger + + +log = MyLogger().get_logger() + +if __name__ == "__main__": + print("str.pdf"["str.pdf".rindex(".") :]) + + def test(): + try: + print(3) + except ZeroDivisionError as e: + log.exception(e) + + test() diff --git a/src/agent/web/__init__.py b/src/agent/web/__init__.py new file mode 100644 index 0000000..402d801 --- /dev/null +++ b/src/agent/web/__init__.py @@ -0,0 +1 @@ +"""Web 前端静态资源与本地 API 服务（见 ``server`` 模块）。""" diff --git a/src/agent/web/__pycache__/__init__.cpython-312.pyc b/src/agent/web/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..df896fb Binary files /dev/null and b/src/agent/web/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/web/__pycache__/server.cpython-312.pyc b/src/agent/web/__pycache__/server.cpython-312.pyc new file mode 100644 index 0000000..2e66f19 Binary files /dev/null and b/src/agent/web/__pycache__/server.cpython-312.pyc differ diff --git a/src/agent/web/__pycache__/ui_session.cpython-312.pyc b/src/agent/web/__pycache__/ui_session.cpython-312.pyc new file mode 100644 index 0000000..0e461d7 Binary files /dev/null and b/src/agent/web/__pycache__/ui_session.cpython-312.pyc differ diff --git a/src/agent/web/app.css b/src/agent/web/app.css new file mode 100644 index 0000000..babcdac --- /dev/null +++ b/src/agent/web/app.css @@ -0,0 +1,1269 @@ +:root { + --text: #0f172a; + --muted: #475569; + --accent: #0e7490; + --accent-hover: #155e75; + --radius: 14px; + --sidebar-w: 272px; + --rail-w: 44px; + /* 背景：景深模糊（px），由 JS 写入 html 行内或仍用默认 */ + --bg-dof: 0px; + --bg-vignette: 0.35; + --bubble-user: rgba(236, 254, 255, 0.55); + --bubble-assistant: rgba(255, 255, 255, 0.5); + --bubble-audio-only: rgba(255, 255, 255, 0.22); + --audio-accent: #14b8a6; + --audio-bubble-min: 148px; + --audio-bubble-max: 312px; +} + +* { + box-sizing: border-box; +} + +html { + -webkit-text-size-adjust: 100%; + text-size-adjust: 100%; +} + +html, +body { + margin: 0; + min-height: 100%; + min-height: 100dvh; + font-family: "ZCOOL KuaiLe", "PingFang SC", "Hiragino Sans GB", "Microsoft YaHei", ui-sans-serif, system-ui, sans-serif; + background: transparent; + color: var(--text); +} + +body { + overflow-x: hidden; +} + +/* 全屏背景 + 景深（对底图做轻微放大以裁掉模糊边缘） */ +.bg-scene { + position: fixed; + inset: 0; + z-index: 0; + overflow: hidden; + pointer-events: none; +} + +.bg-image { + position: absolute; + inset: 0; + /* 默认 cover；加载后 app.js 按 natural 尺寸与视口写死 px，等价 cover + 景深 bleed */ + background-color: #334155; + background-image: url("/image/background.jpg"); + background-size: cover; + background-position: center; + background-repeat: no-repeat; + filter: blur(var(--bg-dof)); + will-change: filter, background-size; +} + +.bg-vignette { + position: absolute; + inset: 0; + opacity: var(--bg-vignette); + background: radial-gradient(ellipse 90% 85% at 50% 45%, transparent 0%, rgba(15, 23, 42, 0.92) 100%); + mix-blend-mode: multiply; +} + +.app-root { + position: relative; + z-index: 1; + display: flex; + min-height: 100vh; + min-height: 100dvh; + align-items: stretch; +} + +.drawer-backdrop { + display: none; + position: fixed; + inset: 0; + z-index: 40; + background: rgba(15, 23, 42, 0.45); + -webkit-tap-highlight-color: transparent; +} + +.app-root.drawer-open .drawer-backdrop { + display: block; +} + +.menu-btn { + display: none; + flex-shrink: 0; + width: 2.35rem; + height: 2.35rem; + margin: 0 0.5rem 0 0; + padding: 0; + align-items: center; + justify-content: center; + font-size: 1.15rem; + line-height: 1; + cursor: pointer; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.5); + background: rgba(255, 255, 255, 0.45); + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); + color: #334155; + -webkit-tap-highlight-color: transparent; +} + +.menu-btn:active { + transform: scale(0.97); +} + +.expand-rail { + width: var(--rail-w); + min-width: var(--rail-w); + border-right: 1px solid rgba(148, 163, 184, 0.35); + background: transparent; + display: none; + flex-direction: column; + align-items: center; + justify-content: center; + align-self: stretch; + padding: 0.5rem 0.25rem; + flex-shrink: 0; +} + +.app-root.sidebar-collapsed .expand-rail { + display: flex; +} + +.app-root.sidebar-collapsed .sidebar { + display: none; +} + +.rail-btn, +.collapse-fab { + width: 26px; + min-width: 26px; + height: 52px; + padding: 0; + border-radius: 0 14px 14px 0; + border: 1px solid rgba(148, 163, 184, 0.45); + border-left: none; + background: rgba(255, 255, 255, 0.35); + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + box-shadow: 2px 0 16px rgba(15, 23, 42, 0.08); + font-size: 1rem; + color: #334155; + cursor: pointer; +} + +.rail-btn:hover, +.collapse-fab:hover { + background: rgba(255, 255, 255, 0.5); +} + +.sidebar { + width: var(--sidebar-w); + min-width: var(--sidebar-w); + border-right: 1px solid rgba(148, 163, 184, 0.35); + background: transparent; + display: flex; + flex-direction: column; + padding: 0.65rem 0 0.65rem 0.65rem; + max-height: 100vh; + max-height: 100dvh; + position: relative; +} + +.sidebar-head { + flex-shrink: 0; + padding-right: 0.45rem; +} + +.sidebar-title { + margin: 0 0 0.35rem; + font-size: 0.9rem; + font-weight: 600; + color: #334155; + text-shadow: 0 1px 2px rgba(255, 255, 255, 0.6); +} + +.sidebar-list-wrap { + flex: 1; + min-height: 0; + overflow-y: auto; + padding-right: 0.35rem; +} + +.session-list { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.session-item { + font-size: 0.78rem; + word-break: break-all; + border-radius: 10px; + padding: 0.35rem 0.5rem; + border: 1px solid transparent; + cursor: pointer; + color: var(--text); + background: transparent; + transition: background 0.15s ease, border-color 0.15s ease; +} + +.session-item:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(148, 163, 184, 0.4); +} + +.session-item.active { + border-color: rgba(14, 116, 144, 0.65); + background: rgba(236, 254, 255, 0.35); +} + +.sidebar-hint { + font-size: 0.72rem; + color: var(--muted); + line-height: 1.45; + margin: 0.35rem 0.45rem 0 0; + flex-shrink: 0; + text-shadow: 0 1px 1px rgba(255, 255, 255, 0.5); +} + +.sidebar-bg-panel { + flex-shrink: 0; + margin: 0.6rem 0.45rem 0 0; + padding: 0.55rem 0.5rem; + border-radius: var(--radius); + border: 1px solid rgba(148, 163, 184, 0.22); + background: transparent; +} + +.sidebar-bg-title { + margin: 0 0 0.45rem; + font-size: 0.72rem; + font-weight: 600; + letter-spacing: 0.02em; + color: #334155; + text-transform: uppercase; + opacity: 0.9; +} + +.bg-control { + display: grid; + grid-template-columns: 4.5rem 1fr auto; + align-items: center; + gap: 0.35rem 0.5rem; + margin-bottom: 0.45rem; + font-size: 0.72rem; + color: var(--muted); +} + +.bg-control:last-of-type { + margin-bottom: 0.35rem; +} + +.bg-control-label { + color: #475569; +} + +.bg-control input[type="range"] { + width: 100%; + accent-color: var(--accent); + min-width: 0; +} + +.bg-control-value { + font-variant-numeric: tabular-nums; + color: #334155; + min-width: 3.2rem; + text-align: right; +} + +.sidebar-bg-tip { + margin: 0; + font-size: 0.65rem; + line-height: 1.4; + color: #64748b; +} + +.sidebar-bg-tip code { + font-size: 0.62rem; + background: rgba(255, 255, 255, 0.35); + padding: 0.1rem 0.25rem; + border-radius: 4px; +} + +.sidebar-collapse { + position: absolute; + right: 0; + top: 50%; + transform: translateY(-50%); + width: 34px; + display: flex; + justify-content: center; +} + +.collapse-fab { + border-radius: 14px 0 0 14px; + border-right: none; + border-left: 1px solid rgba(148, 163, 184, 0.45); + box-shadow: -2px 0 16px rgba(15, 23, 42, 0.08); +} + +.main { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + padding: 0.75rem 1rem 1rem; + padding-bottom: max(1rem, env(safe-area-inset-bottom, 0px)); + max-height: 100vh; + max-height: 100dvh; + overflow: hidden; + background: transparent; +} + +.main-header { + flex-shrink: 0; + margin-bottom: 0.5rem; + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.65rem; +} + +.main-header-text { + flex: 1; + min-width: 0; +} + +.header-toggle { + flex-shrink: 0; + width: 2.1rem; + height: 2.1rem; + margin: 0; + padding: 0; + display: inline-flex; + align-items: center; + justify-content: center; + font-size: 0.95rem; + line-height: 1; + cursor: pointer; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.5); + background: rgba(255, 255, 255, 0.45); + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); + color: #334155; + transition: background 0.15s ease, transform 0.15s ease; +} + +.header-toggle:hover { + background: rgba(255, 255, 255, 0.65); +} + +.main.header-collapsed .main-header-text { + display: none; +} + +.main.header-collapsed .main-header { + margin-bottom: 0.35rem; + justify-content: flex-end; +} + +.title { + margin: 0; + font-size: 1.35rem; + font-weight: 700; + letter-spacing: -0.02em; + color: #0f172a; + text-shadow: 0 1px 3px rgba(255, 255, 255, 0.75); +} + +.subtitle { + margin: 0.2rem 0 0; + font-size: 0.88rem; + color: var(--muted); + text-shadow: 0 1px 2px rgba(255, 255, 255, 0.65); +} + +.chat-shell { + flex: 1; + min-height: 0; + display: flex; + flex-direction: column; + border: 1px solid rgba(148, 163, 184, 0.4); + border-radius: var(--radius); + background: transparent; + overflow: hidden; + box-shadow: 0 8px 32px rgba(15, 23, 42, 0.06); +} + +.chat-placeholder { + padding: 1.35rem 1.25rem; + color: var(--muted); + background: transparent; +} + +.chat-placeholder h3 { + margin: 0 0 0.5rem; + color: #0f172a; + font-weight: 600; + text-shadow: 0 1px 2px rgba(255, 255, 255, 0.7); +} + +.chat-placeholder .muted { + font-size: 0.85rem; + text-shadow: 0 1px 1px rgba(255, 255, 255, 0.55); +} + +.chat-messages { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 0.85rem; + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 0.7rem; + background: transparent; +} + +.msg { + max-width: 95%; + width: fit-content; + min-width: 0; + display: flex; + flex-direction: column; + gap: 0.35rem; +} + +.msg.user { + align-self: flex-end; + align-items: flex-end; +} + +.msg.assistant { + align-self: flex-start; +} + +.msg-bubble { + border-radius: 14px; + padding: 0.6rem 0.85rem; + border: 1px solid rgba(148, 163, 184, 0.45); + line-height: 1.55; + white-space: pre-wrap; + word-break: break-word; + width: fit-content; + max-width: 100%; + box-sizing: border-box; + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + box-shadow: 0 4px 20px rgba(15, 23, 42, 0.06); +} + +.msg.user .msg-bubble { + background: var(--bubble-user); +} + +.msg.assistant .msg-bubble { + background: var(--bubble-assistant); +} + +.msg-bubble img { + display: block; + width: auto; + height: auto; + max-width: min(360px, 100%); + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.35); + margin-top: 0.25rem; +} + +.msg-bubble > img:first-child { + margin-top: 0; +} + + .msg-bubble audio { + max-width: 100%; + display: block; + margin-top: 0.25rem; +} + +.msg--audio-only { + max-width: min(var(--audio-bubble-max), 82%); +} + +.msg--audio-only .msg-bubble { + padding: 0.2rem 0; + border: none; + background: transparent; + box-shadow: none; + backdrop-filter: none; + -webkit-backdrop-filter: none; +} + +.audio-pill-wrap { + display: flex; + align-items: center; + width: 100%; +} + +.audio-pill-wrap--assistant { + justify-content: flex-start; +} + +.audio-pill-wrap--user { + justify-content: flex-end; +} + +.audio-pill-wrap--compact { + min-width: var(--audio-bubble-min); +} + +.audio-pill { + position: relative; + overflow: visible; + display: inline-flex; + align-items: center; + justify-content: space-between; + gap: 0.9rem; + min-width: var(--audio-bubble-min); + max-width: min(var(--audio-bubble-max), 100%); + padding: 0.78rem 1rem; + border-radius: 999px; + border: 1px solid rgba(14, 116, 144, 0.28); + background: rgba(255, 255, 255, 0.72); + color: #0f172a; + cursor: pointer; + box-shadow: 0 6px 18px rgba(14, 116, 144, 0.08); + transition: background 0.16s ease, box-shadow 0.16s ease, transform 0.16s ease; +} + +.audio-pill--assistant { + --audio-pill-bg: linear-gradient(135deg, rgba(45, 212, 191, 0.34), rgba(255, 255, 255, 0.9)); + --audio-pill-border: rgba(20, 184, 166, 0.38); + --audio-pill-shadow: rgba(20, 184, 166, 0.18); + --audio-pill-progress-color: rgba(20, 184, 166, 0.18); +} + +.audio-pill--user { + --audio-pill-bg: linear-gradient(135deg, rgba(191, 219, 254, 0.55), rgba(255, 255, 255, 0.92)); + --audio-pill-border: rgba(59, 130, 246, 0.3); + --audio-pill-shadow: rgba(59, 130, 246, 0.14); + --audio-pill-progress-color: rgba(59, 130, 246, 0.16); +} + +.audio-pill--user .audio-pill-main { + flex-direction: row-reverse; + justify-content: flex-start; + gap: 0.55rem; +} + +.audio-pill--user .audio-pill-side { + order: -1; + margin-right: 0.15rem; +} + +.msg--audio-only .audio-pill, +.audio-pill--assistant, +.audio-pill--user { + background: var(--audio-pill-bg); + border-color: var(--audio-pill-border); + box-shadow: 0 10px 26px var(--audio-pill-shadow); +} + +.audio-pill-main { + position: relative; + z-index: 2; + display: inline-flex; + align-items: center; + gap: 0.7rem; + min-width: 0; + flex: 1; +} + +.audio-pill-side { + position: relative; + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: flex-end; + flex-shrink: 0; +} + +.audio-pill-progress { + --audio-progress: 0%; + position: absolute; + inset: 0; + width: var(--audio-progress); + max-width: 100%; + border-radius: inherit; + background: linear-gradient(90deg, var(--audio-pill-progress-color), transparent 92%); + pointer-events: none; + z-index: 1; + transition: width 0.08s linear; +} + +.audio-pill-tail { + position: absolute; + top: 50%; + width: 14px; + height: 14px; + border-bottom: 1px solid var(--audio-pill-border); + background: var(--audio-pill-bg); + transform: translateY(-50%) rotate(45deg); + z-index: 0; +} + +.audio-pill--assistant .audio-pill-tail { + left: -5px; + border-left: 1px solid var(--audio-pill-border); +} + +.audio-pill--user .audio-pill-tail { + right: -5px; + border-right: 1px solid var(--audio-pill-border); +} + +.audio-pill:hover { + background: rgba(255, 255, 255, 0.88); +} + +.audio-pill--assistant:hover { + background: linear-gradient(135deg, rgba(45, 212, 191, 0.42), rgba(255, 255, 255, 0.96)); +} + +.audio-pill--user:hover { + background: linear-gradient(135deg, rgba(191, 219, 254, 0.68), rgba(255, 255, 255, 0.98)); +} + +.audio-pill.is-playing { + background: rgba(240, 253, 250, 0.96); + box-shadow: 0 10px 24px rgba(20, 184, 166, 0.16); +} + +.audio-pill--user.is-playing { + background: rgba(239, 246, 255, 0.98); + box-shadow: 0 10px 24px rgba(59, 130, 246, 0.15); +} + +.audio-pill:active { + transform: scale(0.985); +} + +.audio-pill-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 2rem; + height: 2rem; + border-radius: 999px; + background: linear-gradient(165deg, #0d9488, var(--accent)); + color: #fff; + flex-shrink: 0; +} + +.audio-pill--user .audio-pill-icon { + background: linear-gradient(165deg, #60a5fa, #2563eb); +} + +.audio-pill.is-playing .audio-pill-icon { + background: linear-gradient(165deg, #14b8a6, #0f766e); +} + +.audio-pill--user.is-playing .audio-pill-icon { + background: linear-gradient(165deg, #3b82f6, #1d4ed8); +} + +.audio-pill-wave { + display: inline-flex; + align-items: flex-end; + gap: 0.16rem; + height: 1.25rem; + flex-shrink: 0; +} + +.audio-pill-wave span { + display: block; + width: 0.18rem; + border-radius: 999px; + background: rgba(14, 116, 144, 0.78); + transform-origin: center bottom; +} + +.audio-pill--user .audio-pill-wave span { + background: rgba(37, 99, 235, 0.72); +} + +.audio-pill.is-playing .audio-pill-wave span { + animation: audioWave 1.1s ease-in-out infinite; +} + +.audio-pill.is-playing .audio-pill-wave span:nth-child(2) { + animation-delay: 0.08s; +} + +.audio-pill.is-playing .audio-pill-wave span:nth-child(3) { + animation-delay: 0.16s; +} + +.audio-pill.is-playing .audio-pill-wave span:nth-child(4) { + animation-delay: 0.24s; +} + +.audio-pill.is-playing .audio-pill-wave span:nth-child(5) { + animation-delay: 0.32s; +} + +.audio-pill.is-playing .audio-pill-wave span:nth-child(6) { + animation-delay: 0.4s; +} + +.audio-pill-wave span:nth-child(1) { height: 0.5rem; } +.audio-pill-wave span:nth-child(2) { height: 0.9rem; } +.audio-pill-wave span:nth-child(3) { height: 0.65rem; } +.audio-pill-wave span:nth-child(4) { height: 1.05rem; } +.audio-pill-wave span:nth-child(5) { height: 0.72rem; } +.audio-pill-wave span:nth-child(6) { height: 0.42rem; } + +.audio-pill-text { + font-size: 0.9rem; + text-align: left; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.audio-pill-text--muted { + font-size: 0.72rem; + opacity: 0.55; + letter-spacing: 0.01em; +} + +.audio-pill-duration { + font-size: 0.82rem; + color: #0f766e; + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +.audio-pill--user .audio-pill-duration { + font-size: 0.92rem; + font-weight: 700; + color: #1d4ed8; +} + +.msg--audio-only .audio-pill-duration { + color: #115e59; +} + +.msg-bubble.msg-bubble--rich { + white-space: normal; +} + + .msg-prose { + margin: 0.4rem 0; + font-size: 0.95rem; +} + +.markdown-body { + line-height: 1.65; +} + + .markdown-body p, +.markdown-body ul, +.markdown-body ol, +.markdown-body blockquote, +.markdown-body h1, +.markdown-body h2, +.markdown-body h3, +.markdown-body h4, +.markdown-body h5, +.markdown-body h6 { + margin: 0.45rem 0; +} + +.markdown-body--streaming { + opacity: 0.98; +} + +.markdown-body--streaming > :last-child { + position: relative; +} + +.markdown-body ul, +.markdown-body ol { + padding-left: 1.3rem; +} + +.markdown-body blockquote { + margin-left: 0; + padding-left: 0.85rem; + border-left: 3px solid rgba(14, 116, 144, 0.35); + color: #334155; +} + +.markdown-body code:not(pre code) { + padding: 0.1rem 0.28rem; + border-radius: 6px; + background: rgba(148, 163, 184, 0.18); + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 0.88em; +} + +.msg-prose:first-child { + margin-top: 0; +} + +.msg-prose:last-child { + margin-bottom: 0; +} + + .msg-typing .msg-typing-bubble { + min-width: 5.5rem; + padding: 0.65rem 1rem; +} + +.thinking-label { + font-size: 0.86rem; + margin-bottom: 0.2rem; + color: #334155; +} + +.typing-dots { + display: inline-flex; + gap: 0.2rem; + align-items: center; + font-size: 1.35rem; + line-height: 1; + color: var(--muted); + font-weight: 700; +} + +.typing-dot { + display: inline-block; + animation: typingBounce 1.05s ease-in-out infinite; +} + +.typing-dot:nth-child(2) { + animation-delay: 0.15s; +} + +.typing-dot:nth-child(3) { + animation-delay: 0.3s; +} + +@keyframes audioWave { + 0%, + 100% { + transform: scaleY(0.65); + opacity: 0.72; + } + + 50% { + transform: scaleY(1.15); + opacity: 1; + } +} + +@keyframes typingBounce { + 0%, + 60%, + 100% { + opacity: 0.25; + transform: translateY(0); + } + + 30% { + opacity: 1; + transform: translateY(-3px); + } +} + +.code-block-wrap { + margin: 0.55rem 0; + border-radius: 10px; + overflow: hidden; + border: 1px solid rgba(51, 65, 85, 0.65); + background: rgba(15, 23, 42, 0.94); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.06); +} + +.code-block-wrap:first-child { + margin-top: 0; +} + +.code-block-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.5rem; + padding: 0.35rem 0.55rem; + background: rgba(30, 41, 59, 0.95); + border-bottom: 1px solid rgba(71, 85, 105, 0.6); +} + +.code-block-lang { + font-size: 0.72rem; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + color: #94a3b8; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; +} + +.code-block-copy { + font-size: 0.72rem; + padding: 0.2rem 0.55rem; + border-radius: 6px; + border: 1px solid rgba(148, 163, 184, 0.45); + background: rgba(51, 65, 85, 0.6); + color: #e2e8f0; + cursor: pointer; +} + +.code-block-copy:hover { + background: rgba(71, 85, 105, 0.85); +} + +.code-block-wrap pre { + margin: 0; + padding: 0.65rem 0.75rem; + overflow-x: auto; + max-height: min(55vh, 420px); + background: rgba(15, 23, 42, 0.98); +} + +.code-block-wrap code { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 0.8rem; + line-height: 1.5; + color: #e2e8f0; + white-space: pre; +} + +.composer { + flex-shrink: 0; + margin-top: 0.65rem; + border: 1px solid rgba(148, 163, 184, 0.4); + border-radius: var(--radius); + background: transparent; + padding: 0.55rem; + box-shadow: 0 8px 28px rgba(15, 23, 42, 0.05); +} + +.composer-attachments { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin-bottom: 0.35rem; + min-height: 0; +} + +.chip { + font-size: 0.75rem; + padding: 0.22rem 0.5rem; + border-radius: 999px; + background: rgba(255, 255, 255, 0.45); + border: 1px solid rgba(148, 163, 184, 0.45); + display: inline-flex; + align-items: center; + gap: 0.25rem; + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); +} + +.chip button { + border: none; + background: none; + cursor: pointer; + padding: 0; + font-size: 1rem; + line-height: 1; + color: var(--muted); +} + +.composer-row { + display: flex; + gap: 0.55rem; + align-items: flex-end; +} + +.mm-input { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + gap: 0.35rem; +} + +.composer-input-shell { + position: relative; +} + +.mm-input textarea { + width: 100%; + resize: vertical; + min-height: 52px; + max-height: 220px; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.5); + padding: 0.5rem 3.2rem 0.5rem 0.6rem; + font: inherit; + background: rgba(255, 255, 255, 0.35); + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + color: var(--text); +} + +.mic-btn { + position: absolute; + right: 0.5rem; + bottom: 0.5rem; + width: 2.2rem; + height: 2.2rem; + padding: 0; + display: inline-flex; + align-items: center; + justify-content: center; + border-radius: 999px; + border: 1px solid rgba(14, 116, 144, 0.28); + background: rgba(255, 255, 255, 0.72); + backdrop-filter: blur(10px); + -webkit-backdrop-filter: blur(10px); + cursor: pointer; + transition: transform 0.12s ease, background 0.15s ease, border-color 0.15s ease, box-shadow 0.15s ease; + box-shadow: 0 6px 16px rgba(14, 116, 144, 0.12); +} + +.mic-btn:hover:not(:disabled) { + background: rgba(255, 255, 255, 0.92); +} + +.mic-btn:active:not(:disabled) { + transform: scale(0.96); +} + +.mic-btn:disabled { + opacity: 0.55; + cursor: not-allowed; +} + +.mic-btn img { + width: 1.15rem; + height: 1.15rem; + display: block; +} + +.mic-btn.is-recording { + border-color: rgba(220, 38, 38, 0.45); + background: rgba(254, 226, 226, 0.92); + box-shadow: 0 0 0 4px rgba(239, 68, 68, 0.14); +} + +.mic-btn.is-recording img { + filter: hue-rotate(315deg) saturate(1.2); +} + +.mm-input textarea::placeholder { + color: #64748b; +} + +.mm-input textarea:focus { + outline: 2px solid rgba(14, 116, 144, 0.35); + outline-offset: 1px; +} + +.mm-actions { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + align-items: center; +} + +.composer-send { + display: flex; + flex-direction: column; + gap: 0.35rem; +} + +.btn { + font: inherit; + cursor: pointer; + border-radius: 10px; + padding: 0.42rem 0.72rem; + border: 1px solid rgba(148, 163, 184, 0.5); + background: rgba(255, 255, 255, 0.4); + backdrop-filter: blur(8px); + -webkit-backdrop-filter: blur(8px); + color: var(--text); + transition: background 0.15s ease, border-color 0.15s ease, transform 0.1s ease; +} + +.btn:active { + transform: scale(0.98); +} + +.btn-primary { + background: linear-gradient(165deg, #0d9488, var(--accent)); + border-color: rgba(21, 94, 117, 0.85); + color: #fff; + box-shadow: 0 4px 14px rgba(14, 116, 144, 0.35); +} + +.btn-primary:hover:not(:disabled) { + background: linear-gradient(165deg, #14b8a6, var(--accent-hover)); +} + +.btn-primary:disabled { + opacity: 0.55; + cursor: not-allowed; +} + + .btn-secondary { + background: rgba(255, 255, 255, 0.35); +} + +.btn-danger { + background: rgba(239, 68, 68, 0.15); + border-color: rgba(220, 38, 38, 0.4); + color: #991b1b; +} + +.btn-danger:hover:not(:disabled) { + background: rgba(239, 68, 68, 0.22); +} + +.btn-ghost { + background: rgba(255, 255, 255, 0.2); +} + + .file-btn { + display: inline-flex; + align-items: center; + gap: 0.25rem; + font-size: 0.85rem; +} + +.output-mode-label { + font-size: 0.82rem; + color: #334155; +} + +.output-mode-select { + min-height: 36px; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.5); + padding: 0.35rem 0.55rem; + font: inherit; + background: rgba(255, 255, 255, 0.42); + color: var(--text); +} + +/* —— 窄屏 / 手机：侧栏改为抽屉，主区全宽，避免 272px 挤爆布局 —— */ +@media (max-width: 768px) { + .main-header { + align-items: center; + } + + .menu-btn { + display: inline-flex; + } + + .app-root.is-mobile .sidebar { + position: fixed; + z-index: 50; + left: 0; + top: 0; + bottom: 0; + width: min(300px, 88vw); + min-width: 0; + max-height: none; + height: 100vh; + height: 100dvh; + padding-top: max(0.65rem, env(safe-area-inset-top, 0px)); + padding-left: max(0.65rem, env(safe-area-inset-left, 0px)); + padding-bottom: max(0.65rem, env(safe-area-inset-bottom, 0px)); + margin: 0; + border-right: 1px solid rgba(148, 163, 184, 0.45); + background: rgba(248, 250, 252, 0.92); + backdrop-filter: blur(14px); + -webkit-backdrop-filter: blur(14px); + box-shadow: 8px 0 32px rgba(15, 23, 42, 0.12); + transform: translateX(-105%); + transition: transform 0.22s ease; + overflow-y: auto; + -webkit-overflow-scrolling: touch; + } + + .app-root.is-mobile.drawer-open .sidebar { + transform: translateX(0); + } + + .app-root.is-mobile .expand-rail { + display: none !important; + } + + .app-root.is-mobile.sidebar-collapsed .sidebar { + display: flex; + } + + .app-root.is-mobile .main { + width: 100%; + padding-left: max(0.65rem, env(safe-area-inset-left, 0px)); + padding-right: max(0.65rem, env(safe-area-inset-right, 0px)); + } + + .composer-row { + flex-direction: column; + align-items: stretch; + } + + .composer-send { + flex-direction: row; + justify-content: stretch; + gap: 0.5rem; + } + + .composer-send .btn { + flex: 1; + min-height: 44px; + } + + .mm-input textarea { + font-size: 16px; + min-height: 44px; + padding-right: 3rem; + } + + .mic-btn { + width: 2.4rem; + height: 2.4rem; + right: 0.45rem; + bottom: 0.45rem; + } + + .mm-actions .btn, + .mm-actions .file-btn { + min-height: 40px; + } + + .title { + font-size: 1.15rem; + } + + .subtitle { + font-size: 0.8rem; + } + + .chat-messages { + -webkit-overflow-scrolling: touch; + } +} diff --git a/src/agent/web/app.js b/src/agent/web/app.js new file mode 100644 index 0000000..bf9e8d7 --- /dev/null +++ b/src/agent/web/app.js @@ -0,0 +1,1193 @@ +(() => { + const $ = (id) => document.getElementById(id); + + const sessionList = $("sessionList"); + const chatMessages = $("chatMessages"); + const chatPlaceholder = $("chatPlaceholder"); + const textInput = $("textInput"); + const imageInput = $("imageInput"); + const audioInput = $("audioInput"); + const micBtn = $("micBtn"); + const attachmentPreview = $("attachmentPreview"); + const sendBtn = $("sendBtn"); + const clearBtn = $("clearBtn"); + const newSessionBtn = $("newSessionBtn"); + const deleteSessionBtn = $("deleteSessionBtn"); + const collapseBtn = $("collapseBtn"); + const expandBtn = $("expandBtn"); + const expandRail = $("expandRail"); + const sidebar = $("sidebar"); + const appRoot = $("app"); + const mainPanel = $("mainPanel"); + const headerToggleBtn = $("headerToggleBtn"); + const menuBtn = $("menuBtn"); + const drawerBackdrop = $("drawerBackdrop"); + const bgImageLayer = $("bgImageLayer"); + const outputModeSelect = $("outputModeSelect"); + const dofRange = $("dofRange"); + const dofValue = $("dofValue"); + const vignetteRange = $("vignetteRange"); + const vignetteValue = $("vignetteValue"); + + const STORAGE_DOF = "minor_web_bg_dof"; + const STORAGE_VIG = "minor_web_bg_vignette"; + const STORAGE_HEADER_COLLAPSED = "minor_web_header_collapsed"; + + /** @type {string} */ + let sessionId = ""; + /** @type {string[]} */ + let sessions = []; + /** @type {File[]} */ + let pendingImages = []; + /** @type {File | null} */ + let pendingAudio = null; + let sidebarCollapsed = false; + let sending = false; + let mediaRecorder = null; + let recorderStream = null; + let recordedChunks = []; + let recordingMimeType = ""; + /** 最近一次 renderMessages 的 JSON 可序列化快照，用于发送前拼接带 blob: 的乐观用户消息 */ + let lastRenderedMessages = []; + let activeAudioController = null; + + async function api(path, opts = {}) { + const r = await fetch(path, { + ...opts, + headers: opts.body instanceof FormData ? {} : { "Content-Type": "application/json", ...(opts.headers || {}) }, + }); + if (!r.ok) { + let detail = r.statusText; + try { + const j = await r.json(); + if (j.detail) detail = typeof j.detail === "string" ? j.detail : JSON.stringify(j.detail); + } catch { + /* ignore */ + } + throw new Error(detail || `HTTP ${r.status}`); + } + return r.json(); + } + + function setSidebarCollapsed(collapsed) { + sidebarCollapsed = collapsed; + appRoot.classList.toggle("sidebar-collapsed", collapsed); + expandRail.setAttribute("aria-hidden", collapsed ? "false" : "true"); + } + + function setDrawerOpen(open) { + appRoot.classList.toggle("drawer-open", open); + if (menuBtn) menuBtn.setAttribute("aria-expanded", open ? "true" : "false"); + if (drawerBackdrop) drawerBackdrop.setAttribute("aria-hidden", open ? "false" : "true"); + } + + function updateMobileLayout() { + const mobile = window.matchMedia("(max-width: 768px)").matches; + appRoot.classList.toggle("is-mobile", mobile); + if (!mobile) setDrawerOpen(false); + } + + function closeDrawerIfMobile() { + if (appRoot.classList.contains("is-mobile")) setDrawerOpen(false); + } + + function renderSessions() { + sessionList.innerHTML = ""; + sessions.forEach((id) => { + const li = document.createElement("li"); + li.className = "session-item" + (id === sessionId ? " active" : ""); + li.textContent = id; + li.title = id; + li.setAttribute("role", "option"); + li.setAttribute("aria-selected", id === sessionId ? "true" : "false"); + li.addEventListener("click", () => selectSession(id)); + sessionList.appendChild(li); + }); + } + + function escapeHtml(s) { + return String(s) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); + } + + /** 与后端气泡结构一致：纯文字为 string，否则为 parts 数组（顺序：图片 → 文字 → 音频） */ + function buildOptimisticUserContent(text, imageBlobUrls, audioPart) { + const parts = []; + (imageBlobUrls || []).forEach((url) => { + parts.push({ type: "image", url }); + }); + const t = (text || "").trim(); + if (t) parts.push({ type: "text", text: t }); + if (audioPart && audioPart.url) { + parts.push({ type: "audio", url: audioPart.url, name: audioPart.name || "音频" }); + } + if (parts.length === 0) return null; + if (parts.length === 1 && parts[0].type === "text") return parts[0].text; + return parts; + } + + function getTextFromContent(content) { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content + .filter((part) => part && part.type === "text" && typeof part.text === "string") + .map((part) => part.text) + .join("\n") + .trim(); + } + + function getMediaParts(content) { + if (!Array.isArray(content)) return []; + return content.filter((part) => part && part.type !== "text"); + } + + function isAudioOnlyAssistantMessage(msg) { + if (!msg || msg.role !== "assistant") return false; + const outputType = msg.meta && typeof msg.meta.output_type === "string" ? msg.meta.output_type : ""; + const text = getTextFromContent(msg.content); + const mediaParts = getMediaParts(msg.content); + const hasOnlyAudioMedia = mediaParts.length > 0 && mediaParts.every((part) => part.type === "audio" && part.url); + return (outputType === "audio" || !text) && hasOnlyAudioMedia && !text; + } + + function clamp(n, min, max) { + return Math.max(min, Math.min(max, n)); + } + + function formatAudioDuration(seconds) { + const n = Number(seconds); + if (!Number.isFinite(n) || n <= 0) return ""; + if (n < 10) return `${n.toFixed(1)}″`; + return `${Math.round(n)}″`; + } + + function getAudioBubbleWidth(seconds) { + const n = Number(seconds); + if (!Number.isFinite(n) || n <= 0) return null; + const width = 132 + Math.sqrt(n) * 46; + return `${Math.round(clamp(width, 148, 312))}px`; + } + + function revokeBlobUrlStack(urlStack) { + (urlStack || []).forEach((u) => { + try { + URL.revokeObjectURL(u); + } catch { + /* ignore */ + } + }); + urlStack.length = 0; + } + + function stopRecorderStream() { + if (!recorderStream) return; + recorderStream.getTracks().forEach((track) => { + try { + track.stop(); + } catch { + /* ignore */ + } + }); + recorderStream = null; + } + + function setRecordingUi(recording) { + if (!micBtn) return; + micBtn.classList.toggle("is-recording", recording); + micBtn.setAttribute("aria-pressed", recording ? "true" : "false"); + micBtn.title = recording ? "停止录音" : "使用麦克风录音"; + micBtn.setAttribute("aria-label", recording ? "停止录音" : "使用麦克风录音"); + } + + function inferRecordedExtension(mimeType) { + const mt = String(mimeType || "").toLowerCase(); + if (mt.includes("wav")) return ".wav"; + if (mt.includes("mpeg")) return ".mp3"; + if (mt.includes("mp4") || mt.includes("mpeg4")) return ".m4a"; + if (mt.includes("ogg")) return ".ogg"; + if (mt.includes("webm")) return ".webm"; + return ".wav"; + } + + function buildRecordedAudioFile(blob, mimeType) { + const ext = inferRecordedExtension(mimeType || blob.type); + const normalizedType = mimeType || blob.type || (ext === ".wav" ? "audio/wav" : "audio/webm"); + return new File([blob], `microphone${ext}`, { type: normalizedType }); + } + + async function stopMicRecording() { + if (!mediaRecorder) return; + const recorder = mediaRecorder; + const stopped = new Promise((resolve, reject) => { + recorder.addEventListener("stop", () => { + try { + const blob = new Blob(recordedChunks, { type: recordingMimeType || recorder.mimeType || "audio/webm" }); + pendingAudio = buildRecordedAudioFile(blob, recordingMimeType || recorder.mimeType); + renderAttachmentChips(); + resolve(); + } catch (err) { + reject(err); + } finally { + recordedChunks = []; + recordingMimeType = ""; + mediaRecorder = null; + stopRecorderStream(); + setRecordingUi(false); + } + }, { once: true }); + recorder.addEventListener("error", (event) => { + recordedChunks = []; + recordingMimeType = ""; + mediaRecorder = null; + stopRecorderStream(); + setRecordingUi(false); + reject(event.error || new Error("录音失败")); + }, { once: true }); + }); + recorder.stop(); + await stopped; + } + + async function startMicRecording() { + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia || typeof MediaRecorder === "undefined") { + throw new Error("当前浏览器不支持麦克风录音"); + } + if (sending) return; + + const preferredTypes = [ + "audio/wav", + "audio/mp4", + "audio/ogg;codecs=opus", + "audio/ogg", + "audio/webm;codecs=opus", + "audio/webm", + ]; + const supportedType = preferredTypes.find((type) => { + try { + return MediaRecorder.isTypeSupported(type); + } catch { + return false; + } + }) || ""; + + recorderStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + recordedChunks = []; + recordingMimeType = supportedType; + mediaRecorder = supportedType + ? new MediaRecorder(recorderStream, { mimeType: supportedType }) + : new MediaRecorder(recorderStream); + recordingMimeType = mediaRecorder.mimeType || supportedType; + + mediaRecorder.addEventListener("dataavailable", (event) => { + if (event.data && event.data.size > 0) { + recordedChunks.push(event.data); + } + }); + mediaRecorder.start(); + setRecordingUi(true); + } + + async function toggleMicRecording() { + if (!micBtn) return; + micBtn.disabled = true; + try { + if (mediaRecorder && mediaRecorder.state === "recording") { + await stopMicRecording(); + } else { + await startMicRecording(); + } + } finally { + micBtn.disabled = false; + } + } + + function scrollChatToBottom() { + chatMessages.scrollTop = chatMessages.scrollHeight; + } + + function escapeHtmlPreserveWhitespace(text) { + const s = String(text ?? ""); + return escapeHtml(s) + .replace(/ /g, " ") + .replace(/\t/g, "    ") + .replace(/\n/g, "
"); + } + + function renderMarkdownToFragment(text) { + const raw = String(text ?? ""); + const html = window.marked + ? window.marked.parse(raw, { breaks: true, gfm: true }) + : escapeHtmlPreserveWhitespace(raw); + const safeHtml = window.DOMPurify ? window.DOMPurify.sanitize(html) : html; + const tpl = document.createElement("template"); + tpl.innerHTML = safeHtml; + + tpl.content.querySelectorAll("pre code").forEach((codeEl) => { + const pre = codeEl.parentElement; + if (!pre) return; + const rawCode = codeEl.textContent || ""; + const cls = codeEl.className || ""; + const m = cls.match(/language-([\w-]+)/); + const wrap = codeBlockNode(m ? m[1] : "text", rawCode); + pre.replaceWith(wrap); + }); + return tpl.content.cloneNode(true); + } + + function splitMarkdownStreamBlocks(text) { + const raw = String(text ?? ""); + const lines = raw.split("\n"); + const blocks = []; + let current = []; + let inFence = false; + + function flushCurrent() { + if (current.length === 0) return; + blocks.push(current.join("\n")); + current = []; + } + + for (const line of lines) { + const trimmed = line.trimStart(); + const isFence = trimmed.startsWith("```"); + + if (isFence) { + current.push(line); + if (inFence) { + flushCurrent(); + inFence = false; + } else { + flushCurrent(); + inFence = true; + current = [line]; + } + continue; + } + + if (inFence) { + current.push(line); + continue; + } + + if (line.trim() === "") { + current.push(line); + flushCurrent(); + continue; + } + + const looksLikeList = /^\s*([-*+]\s|\d+\.\s|>\s)/.test(line); + if (looksLikeList) { + current.push(line); + continue; + } + + const prev = current[current.length - 1] || ""; + const prevLooksLikeList = /^\s*([-*+]\s|\d+\.\s|>\s)/.test(prev); + if (prevLooksLikeList && line.trim() !== "") { + current.push(line); + continue; + } + + current.push(line); + } + + flushCurrent(); + return blocks; + } + + function renderMarkdownBlock(text, isStreaming = false) { + const raw = String(text ?? ""); + const d = document.createElement("div"); + d.className = "msg-prose markdown-body" + (isStreaming ? " markdown-body--streaming" : ""); + d.appendChild(renderMarkdownToFragment(raw)); + return d; + } + + function codeBlockNode(lang, code) { + const raw = code.replace(/\n$/, ""); + const wrap = document.createElement("div"); + wrap.className = "code-block-wrap"; + const head = document.createElement("div"); + head.className = "code-block-head"; + const langSpan = document.createElement("span"); + langSpan.className = "code-block-lang"; + langSpan.textContent = (lang && lang.trim()) || "text"; + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "code-block-copy"; + btn.textContent = "复制全部"; + btn.addEventListener("click", () => { + navigator.clipboard.writeText(raw).catch(() => {}); + }); + head.appendChild(langSpan); + head.appendChild(btn); + const pre = document.createElement("pre"); + const codeEl = document.createElement("code"); + codeEl.textContent = raw; + pre.appendChild(codeEl); + wrap.appendChild(head); + wrap.appendChild(pre); + return wrap; + } + + /** 解析 ```lang … ``` 与正文，生成片段（用于助手气泡）。 */ + function parseAssistantContent(full) { + return renderMarkdownToFragment(full); + } + + function fillAssistantBubbleImmediate(bubble, text) { + const t = text == null ? "" : String(text); + bubble.classList.add("msg-bubble--rich", "markdown-body"); + bubble.textContent = ""; + bubble.appendChild(renderMarkdownToFragment(t)); + } + + /** 整包到达后：先伪流式显示纯文本，再替换为代码块等富结构。 */ + async function pseudoStreamAssistant(bubble, full) { + const t = full == null ? "" : String(full); + bubble.classList.add("msg-bubble--rich", "markdown-body"); + bubble.textContent = ""; + + const n = t.length; + const charsPerSec = 85; + const totalMs = Math.min(12000, Math.max(400, (n / charsPerSec) * 1000)); + const t0 = performance.now(); + let lastSnapshot = ""; + + return new Promise((resolve) => { + function frame(now) { + const elapsed = now - t0; + const p = Math.min(1, elapsed / totalMs); + const k = Math.min(n, Math.floor(p * n)); + const snapshot = t.slice(0, k); + + if (snapshot !== lastSnapshot) { + lastSnapshot = snapshot; + const blocks = splitMarkdownStreamBlocks(snapshot); + bubble.textContent = ""; + blocks.forEach((block, idx) => { + const isLast = idx === blocks.length - 1; + const stillOpenFence = (block.match(/```/g) || []).length % 2 === 1; + bubble.appendChild(renderMarkdownBlock(block, isLast || stillOpenFence)); + }); + scrollChatToBottom(); + } + + if (p < 1) { + requestAnimationFrame(frame); + } else { + bubble.textContent = ""; + bubble.appendChild(renderMarkdownToFragment(t)); + scrollChatToBottom(); + resolve(); + } + } + requestAnimationFrame(frame); + }); + } + + /** 流式渲染新生成的 assistant 消息（允许自动播放音频） */ + async function streamAssistantMessage(msg) { + const wrap = document.createElement("div"); + wrap.className = "msg assistant" + (isAudioOnlyAssistantMessage(msg) ? " msg--audio-only" : ""); + const bubble = document.createElement("div"); + bubble.className = "msg-bubble"; + wrap.appendChild(bubble); + chatMessages.appendChild(wrap); + chatPlaceholder.hidden = true; + chatMessages.hidden = false; + + const parts = Array.isArray(msg.content) ? msg.content : [{ type: "text", text: String(msg.content || "") }]; + const textParts = parts.filter((part) => part && part.type === "text" && typeof part.text === "string"); + const mediaParts = parts.filter((part) => !part || part.type !== "text"); + + if (textParts.length > 0) { + await pseudoStreamAssistant(bubble, textParts.map((part) => part.text || "").join("\n")); + } + + // 媒体部分：显式开启 allowAutoplay + const mediaOptions = { + autoplay: true, + compact: textParts.length === 0, + messageMeta: msg.meta || null, + role: "assistant", + allowAutoplay: true, + }; + + mediaParts.forEach((part) => { + bubble.appendChild(renderContentPart(part, mediaOptions)); + }); + + if (textParts.length === 0 && mediaParts.length > 0) { + bubble.textContent = ""; + mediaParts.forEach((part) => { + bubble.appendChild(renderContentPart(part, mediaOptions)); + }); + } + + scrollChatToBottom(); + } + + function appendThinkingIndicator() { + const wrap = document.createElement("div"); + wrap.className = "msg assistant msg-typing"; + wrap.setAttribute("aria-live", "polite"); + const bubble = document.createElement("div"); + bubble.className = "msg-bubble msg-typing-bubble"; + const label = document.createElement("div"); + label.className = "thinking-label"; + label.textContent = "思考中..."; + const dots = document.createElement("span"); + dots.className = "typing-dots"; + for (let i = 0; i < 3; i++) { + const d = document.createElement("span"); + d.className = "typing-dot"; + d.textContent = "·"; + dots.appendChild(d); + } + bubble.appendChild(label); + bubble.appendChild(dots); + wrap.appendChild(bubble); + chatMessages.appendChild(wrap); + scrollChatToBottom(); + return wrap; + } + + function renderAudioBubble(part, options = {}) { + const { + autoplay = false, + compact = false, + messageMeta = null, + role = "assistant", + allowAutoplay = false, // 默认不自动播放，只对新生成消息显式开启 + } = options; + + const wrap = document.createElement("div"); + wrap.className = "audio-pill-wrap" + (compact ? " audio-pill-wrap--compact" : "") + (role === "user" ? " audio-pill-wrap--user" : " audio-pill-wrap--assistant"); + + const durationSeconds = part && part.duration_seconds != null + ? part.duration_seconds + : (messageMeta && messageMeta.audio_duration_seconds != null ? messageMeta.audio_duration_seconds : null); + const durationLabel = formatAudioDuration(durationSeconds); + const bubbleWidth = getAudioBubbleWidth(durationSeconds); + const displayName = role === "user" ? "语音" : (part.name || "语音消息"); + + const button = document.createElement("button"); + button.type = "button"; + button.className = `audio-pill audio-pill--${role}`; + if (bubbleWidth) button.style.width = bubbleWidth; + button.innerHTML = ` + + + + ▶ + + ${escapeHtml(displayName)} + + + 0.0″ / ${escapeHtml(durationLabel || "语音")} + + `; + + const audio = new Audio(part.url); + audio.preload = "metadata"; + const iconEl = button.querySelector(".audio-pill-icon"); + const durationEl = button.querySelector(".audio-pill-duration"); + const progressEl = button.querySelector(".audio-pill-progress"); + let resolvedDuration = Number(durationSeconds); + + function updateProgressUi() { + const total = Number.isFinite(resolvedDuration) && resolvedDuration > 0 ? resolvedDuration : audio.duration; + const current = audio.currentTime || 0; + const currentLabel = formatAudioDuration(current) || "0.0″"; + const totalLabel = formatAudioDuration(total) || durationLabel || "语音"; + if (durationEl) durationEl.textContent = role === "user" ? totalLabel : `${currentLabel} / ${totalLabel}`; + if (progressEl && Number.isFinite(total) && total > 0) { + const percent = clamp((current / total) * 100, 0, 100); + progressEl.style.setProperty("--audio-progress", `${percent}%`); + } + } + + function syncDurationFromAudio() { + const runtimeDuration = audio.duration; + if (Number.isFinite(runtimeDuration) && runtimeDuration > 0) { + resolvedDuration = runtimeDuration; + const runtimeWidth = getAudioBubbleWidth(runtimeDuration); + if (runtimeWidth) button.style.width = runtimeWidth; + } + updateProgressUi(); + } + + function setPlayingUi(playing) { + button.classList.toggle("is-playing", playing); + if (iconEl) iconEl.textContent = playing ? "❚❚" : "▶"; + button.setAttribute("aria-pressed", playing ? "true" : "false"); + button.title = playing ? "暂停播放" : "播放语音"; + if (!playing && activeAudioController && activeAudioController.audio === audio) { + activeAudioController = null; + } + } + + audio.addEventListener("loadedmetadata", syncDurationFromAudio); + audio.addEventListener("durationchange", syncDurationFromAudio); + audio.addEventListener("timeupdate", updateProgressUi); + audio.addEventListener("play", () => { + if (activeAudioController && activeAudioController.audio !== audio) { + try { + activeAudioController.audio.pause(); + } catch { + /* ignore */ + } + } + activeAudioController = { audio, button }; + setPlayingUi(true); + updateProgressUi(); + }); + audio.addEventListener("pause", () => { + setPlayingUi(false); + updateProgressUi(); + }); + audio.addEventListener("ended", () => { + audio.currentTime = 0; + setPlayingUi(false); + updateProgressUi(); + }); + + button.addEventListener("click", async () => { + try { + if (!audio.paused) { + audio.pause(); + return; + } + if (activeAudioController && activeAudioController.audio !== audio) { + try { + activeAudioController.audio.pause(); + } catch { + /* ignore */ + } + } + await audio.play(); + } catch { + /* ignore */ + } + }); + + wrap.appendChild(button); + updateProgressUi(); + + // 只有 allowAutoplay 为 true 时才自动播放 + const shouldAutoplay = allowAutoplay !== false && Boolean( + autoplay || part.autoplay || (messageMeta && messageMeta.autoplay) + ); + + if (shouldAutoplay) { + queueMicrotask(async () => { + try { + if (activeAudioController && activeAudioController.audio !== audio) { + try { + activeAudioController.audio.pause(); + } catch { + /* ignore */ + } + } + await audio.play(); + } catch { + setPlayingUi(false); + updateProgressUi(); + } + }); + } + + return wrap; + } + + function renderContentPart(part, options = {}) { + if (typeof part === "string") { + const div = document.createElement("div"); + div.className = "msg-text"; + div.textContent = part; + return div; + } + if (!part || typeof part !== "object") { + const div = document.createElement("div"); + div.className = "msg-text"; + div.textContent = String(part); + return div; + } + if (part.type === "text") { + const div = document.createElement("div"); + div.className = "msg-text"; + div.textContent = part.text || ""; + return div; + } + if (part.type === "image" && part.url) { + const img = document.createElement("img"); + img.src = part.url; + img.alt = "图片"; + return img; + } + if (part.type === "audio" && part.url) { + return renderAudioBubble(part, options); + } + if (part.type === "file" && part.url) { + const a = document.createElement("a"); + a.href = part.url; + a.textContent = part.name || "附件"; + a.target = "_blank"; + a.rel = "noopener"; + return a; + } + const div = document.createElement("div"); + div.className = "msg-text"; + div.textContent = JSON.stringify(part); + return div; + } + + /** + * 渲染单条消息 + * @param {object} msg - 消息对象 + * @param {boolean} allowAutoplay - 是否允许音频自动播放（仅新生成消息为 true） + */ + function renderMessage(msg, allowAutoplay = false) { + const wrap = document.createElement("div"); + wrap.className = "msg " + (msg.role === "user" ? "user" : "assistant") + (isAudioOnlyAssistantMessage(msg) ? " msg--audio-only" : ""); + const bubble = document.createElement("div"); + bubble.className = "msg-bubble"; + const c = msg.content; + + const baseOptions = { + messageMeta: msg.meta || null, + role: msg.role === "user" ? "user" : "assistant", + allowAutoplay: allowAutoplay, + }; + + if (msg.role === "assistant" && typeof c === "string") { + fillAssistantBubbleImmediate(bubble, c); + } else if (typeof c === "string") { + bubble.textContent = c; + } else if (Array.isArray(c)) { + const textParts = c.filter((part) => part && part.type === "text" && typeof part.text === "string"); + const mediaParts = c.filter((part) => !part || part.type !== "text"); + + if (msg.role === "assistant" && textParts.length > 0) { + fillAssistantBubbleImmediate(bubble, textParts.map((part) => part.text || "").join("\n")); + } else { + textParts.forEach((p) => { + bubble.appendChild(renderContentPart(p, baseOptions)); + }); + } + + mediaParts.forEach((p) => { + const partOptions = { + ...baseOptions, + compact: msg.role === "assistant" && textParts.length === 0, + }; + bubble.appendChild(renderContentPart(p, partOptions)); + }); + } else { + bubble.textContent = c == null ? "" : String(c); + } + wrap.appendChild(bubble); + return wrap; + } + + /** + * 渲染消息列表 + * @param {Array} messages - 消息数组 + * @param {boolean} allowAutoplay - 是否允许音频自动播放（默认 false） + */ + function renderMessages(messages, allowAutoplay = false) { + chatMessages.innerHTML = ""; + const list = messages || []; + lastRenderedMessages = list.length ? JSON.parse(JSON.stringify(list)) : []; + const has = list.length > 0; + chatPlaceholder.hidden = has; + chatMessages.hidden = !has; + if (!has) return; + list.forEach((m) => chatMessages.appendChild(renderMessage(m, allowAutoplay))); + scrollChatToBottom(); + } + + function renderAttachmentChips() { + attachmentPreview.innerHTML = ""; + pendingImages.forEach((f, i) => { + const chip = document.createElement("span"); + chip.className = "chip"; + chip.innerHTML = `${escapeHtml(f.name)} `; + chip.querySelector("button").addEventListener("click", () => { + pendingImages = pendingImages.filter((_, j) => j !== i); + renderAttachmentChips(); + }); + attachmentPreview.appendChild(chip); + }); + if (pendingAudio) { + const chip = document.createElement("span"); + chip.className = "chip"; + chip.innerHTML = `${escapeHtml(pendingAudio.name)} `; + chip.querySelector("button").addEventListener("click", () => { + pendingAudio = null; + audioInput.value = ""; + renderAttachmentChips(); + }); + attachmentPreview.appendChild(chip); + } + } + + async function bootstrap() { + const data = await api("/api/bootstrap"); + sessionId = data.sessionId; + sessions = data.sessions || []; + renderSessions(); + renderMessages(data.messages); // allowAutoplay 默认为 false + } + + async function selectSession(id) { + if (!id || id === sessionId) return; + const data = await api(`/api/sessions/${encodeURIComponent(id)}/messages`); + sessionId = data.sessionId; + sessions = data.sessions || []; + renderSessions(); + renderMessages(data.messages); // 切换会话不自动播放 + closeDrawerIfMobile(); + } + + async function newSession() { + const data = await api("/api/sessions/new", { method: "POST" }); + sessionId = data.sessionId; + sessions = data.sessions || []; + pendingImages = []; + pendingAudio = null; + imageInput.value = ""; + audioInput.value = ""; + renderSessions(); + renderAttachmentChips(); + renderMessages(data.messages); // 新建会话不自动播放 + closeDrawerIfMobile(); + } + + async function deleteSession() { + if (!sessionId) return; + if (!window.confirm(`确认删除会话 ${sessionId} 吗？本地保存的图片、音频和该会话记录也会删除。`)) return; + const deletingSessionId = sessionId; + const fd = new FormData(); + fd.append("session_id", sessionId); + const data = await api("/api/sessions/delete", { method: "POST", body: fd }); + sessionId = data.sessionId; + sessions = (data.sessions || []).filter((id) => id !== deletingSessionId); + pendingImages = []; + pendingAudio = null; + imageInput.value = ""; + audioInput.value = ""; + renderAttachmentChips(); + renderSessions(); + renderMessages(data.messages || []); // 删除后不自动播放 + closeDrawerIfMobile(); + } + + async function clearChat() { + if (!sessionId) return; + const fd = new FormData(); + fd.append("session_id", sessionId); + const data = await api("/api/sessions/clear", { method: "POST", body: fd }); + renderMessages(data.messages); // 清空后不自动播放 + } + + async function sendChat() { + if (!sessionId || sending) return; + const trimmed = textInput.value.trim(); + if (!trimmed && pendingImages.length === 0 && !pendingAudio) return; + sending = true; + sendBtn.disabled = true; + + const savedTextRaw = textInput.value; + const savedImages = pendingImages.slice(); + const savedAudio = pendingAudio; + const priorMessages = JSON.parse(JSON.stringify(lastRenderedMessages)); + const blobUrls = []; + const hadOptimisticMedia = savedImages.length > 0 || !!savedAudio; + + if (hadOptimisticMedia) { + const imageBlobUrls = savedImages.map((f) => { + const u = URL.createObjectURL(f); + blobUrls.push(u); + return u; + }); + let audioPart = null; + if (savedAudio) { + const u = URL.createObjectURL(savedAudio); + blobUrls.push(u); + audioPart = { url: u, name: savedAudio.name }; + } + const optimisticContent = buildOptimisticUserContent(trimmed, imageBlobUrls, audioPart); + if (optimisticContent != null) { + renderMessages([...priorMessages, { role: "user", content: optimisticContent }]); // 乐观渲染不自动播放 + } + textInput.value = ""; + pendingImages = []; + pendingAudio = null; + imageInput.value = ""; + audioInput.value = ""; + renderAttachmentChips(); + } + + const fd = new FormData(); + fd.append("session_id", sessionId); + fd.append("text", trimmed); + savedImages.forEach((f) => fd.append("images", f, f.name)); + if (savedAudio) fd.append("audio", savedAudio, savedAudio.name); + let typingEl = null; + try { + const startData = await api("/api/chat/start", { method: "POST", body: fd }); + + sessionId = startData.sessionId; + sessions = startData.sessions || []; + if (!hadOptimisticMedia) { + textInput.value = ""; + pendingImages = []; + pendingAudio = null; + imageInput.value = ""; + audioInput.value = ""; + renderAttachmentChips(); + } + renderSessions(); + renderMessages(startData.messages || []); // start 返回不自动播放 + revokeBlobUrlStack(blobUrls); + typingEl = appendThinkingIndicator(); + + const fd2 = new FormData(); + fd2.append("session_id", sessionId); + fd2.append("output_type", outputModeSelect ? outputModeSelect.value : "text"); + const done = await api("/api/chat/complete", { method: "POST", body: fd2 }); + if (typingEl && typingEl.parentNode) typingEl.remove(); + typingEl = null; + + sessionId = done.sessionId; + sessions = done.sessions || []; + renderSessions(); + + const messages = done.messages || []; + const last = messages[messages.length - 1]; + if (last && last.role === "assistant") { + const head = messages.slice(0, -1); + renderMessages(head); // 历史消息不自动播放 + revokeBlobUrlStack(blobUrls); + await streamAssistantMessage(last); // 只有新消息流式渲染时自动播放 + } else { + renderMessages(messages); // 无 assistant 消息时不自动播放 + revokeBlobUrlStack(blobUrls); + } + } catch (e) { + revokeBlobUrlStack(blobUrls); + if (hadOptimisticMedia) { + textInput.value = savedTextRaw; + pendingImages = savedImages.slice(); + pendingAudio = savedAudio; + renderAttachmentChips(); + renderMessages(priorMessages); // 回滚不自动播放 + } + if (typingEl && typingEl.parentNode) typingEl.remove(); + alert(e.message || String(e)); + } finally { + sending = false; + sendBtn.disabled = false; + } + } + + + imageInput.addEventListener("change", () => { + const files = Array.from(imageInput.files || []); + pendingImages = pendingImages.concat(files); + imageInput.value = ""; + renderAttachmentChips(); + }); + + audioInput.addEventListener("change", () => { + const f = audioInput.files && audioInput.files[0]; + pendingAudio = f || null; + audioInput.value = ""; + renderAttachmentChips(); + }); + + window.addEventListener("beforeunload", () => { + if (mediaRecorder && mediaRecorder.state === "recording") { + try { + mediaRecorder.stop(); + } catch { + /* ignore */ + } + } + stopRecorderStream(); + }); + + textInput.addEventListener("keydown", (ev) => { + if (ev.key === "Enter" && !ev.shiftKey) { + ev.preventDefault(); + sendChat(); + } + }); + + sendBtn.addEventListener("click", sendChat); + micBtn?.addEventListener("click", () => toggleMicRecording().catch((e) => alert(e.message || String(e)))); + clearBtn.addEventListener("click", () => clearChat().catch((e) => alert(e.message))); + newSessionBtn.addEventListener("click", () => newSession().catch((e) => alert(e.message))); + deleteSessionBtn?.addEventListener("click", () => deleteSession().catch((e) => alert(e.message))); + collapseBtn.addEventListener("click", () => { + if (appRoot.classList.contains("is-mobile")) setDrawerOpen(false); + else setSidebarCollapsed(true); + }); + expandBtn.addEventListener("click", () => setSidebarCollapsed(false)); + menuBtn?.addEventListener("click", () => setDrawerOpen(!appRoot.classList.contains("drawer-open"))); + drawerBackdrop?.addEventListener("click", () => setDrawerOpen(false)); + + function applyBgVars() { + const dofPx = dofRange ? String(dofRange.value) : "0"; + const vig = vignetteRange ? Number(vignetteRange.value) / 100 : 0.35; + document.documentElement.style.setProperty("--bg-dof", `${dofPx}px`); + document.documentElement.style.setProperty("--bg-vignette", String(Math.max(0, Math.min(1, vig)))); + if (dofValue) dofValue.textContent = `${dofPx} px`; + if (vignetteValue) vignetteValue.textContent = `${vignetteRange ? vignetteRange.value : 35} %`; + applyBackgroundFit(); + } + + function loadBgPrefs() { + try { + const d = localStorage.getItem(STORAGE_DOF); + if (d != null && dofRange) dofRange.value = d; + const v = localStorage.getItem(STORAGE_VIG); + if (v != null && vignetteRange) vignetteRange.value = v; + } catch { + /* ignore */ + } + applyBgVars(); + } + + if (dofRange) { + dofRange.addEventListener("input", () => { + applyBgVars(); + applyBackgroundFit(); + try { + localStorage.setItem(STORAGE_DOF, dofRange.value); + } catch { + /* ignore */ + } + }); + } + if (vignetteRange) { + vignetteRange.addEventListener("input", () => { + applyBgVars(); + try { + localStorage.setItem(STORAGE_VIG, vignetteRange.value); + } catch { + /* ignore */ + } + }); + } + loadBgPrefs(); + + /** 背景图原始像素（预加载成功后写入） */ + let bgNaturalW = 0; + let bgNaturalH = 0; + /** @type {ReturnType | null} */ + let bgFitTimer = null; + + function backgroundImageHref() { + try { + return new URL("image/background.jpg", window.location.href).href; + } catch { + return "image/background.jpg"; + } + } + + function readDofPx() { + const raw = getComputedStyle(document.documentElement).getPropertyValue("--bg-dof").trim(); + const n = parseFloat(raw); + return Number.isFinite(n) ? n : 0; + } + + /** 按「铺满视口」缩放（等价 cover，无黑边；景深略放大以减轻模糊露边） */ + function applyBackgroundFit() { + if (!bgImageLayer || !bgNaturalW || !bgNaturalH) return; + const vw = window.innerWidth; + const vh = window.innerHeight; + const s = Math.max(vw / bgNaturalW, vh / bgNaturalH); + const dof = readDofPx(); + const bleed = 1 + Math.min(Math.max(dof, 0), 24) * 0.018; + const bw = bgNaturalW * s * bleed; + const bh = bgNaturalH * s * bleed; + bgImageLayer.style.backgroundSize = `${bw}px ${bh}px`; + bgImageLayer.style.backgroundPosition = "center center"; + } + + function scheduleBackgroundFit() { + if (bgFitTimer) clearTimeout(bgFitTimer); + bgFitTimer = setTimeout(() => { + bgFitTimer = null; + applyBackgroundFit(); + }, 100); + } + + /** 与 CSS 中 url("background.jpg") 一致；预加载得到尺寸后再写 background-size */ + function wireBackgroundImage() { + if (!bgImageLayer) return; + const href = backgroundImageHref(); + bgImageLayer.style.backgroundImage = `url(${JSON.stringify(href)})`; + const probe = new Image(); + probe.onload = () => { + bgNaturalW = probe.naturalWidth; + bgNaturalH = probe.naturalHeight; + if (bgNaturalW > 0 && bgNaturalH > 0) applyBackgroundFit(); + }; + probe.onerror = () => { + bgNaturalW = 0; + bgNaturalH = 0; + bgImageLayer.style.backgroundSize = "cover"; + }; + probe.src = href; + } + + function onResizeLayout() { + updateMobileLayout(); + scheduleBackgroundFit(); + } + + window.addEventListener("resize", onResizeLayout); + updateMobileLayout(); + wireBackgroundImage(); + + function setHeaderCollapsed(collapsed) { + if (!mainPanel || !headerToggleBtn) return; + mainPanel.classList.toggle("header-collapsed", collapsed); + headerToggleBtn.setAttribute("aria-expanded", collapsed ? "false" : "true"); + headerToggleBtn.title = collapsed ? "展开标题栏" : "收起标题栏"; + headerToggleBtn.textContent = collapsed ? "▾" : "▴"; + try { + localStorage.setItem(STORAGE_HEADER_COLLAPSED, collapsed ? "1" : "0"); + } catch { + /* ignore */ + } + } + + function loadHeaderPref() { + if (!mainPanel || !headerToggleBtn) return; + try { + if (localStorage.getItem(STORAGE_HEADER_COLLAPSED) === "1") setHeaderCollapsed(true); + } catch { + /* ignore */ + } + } + + headerToggleBtn?.addEventListener("click", () => { + setHeaderCollapsed(!mainPanel.classList.contains("header-collapsed")); + }); + loadHeaderPref(); + + bootstrap().catch((e) => { + chatPlaceholder.hidden = false; + chatPlaceholder.innerHTML = `

无法连接服务

${escapeHtml(e.message)}

请从 Agent 目录执行：uvicorn agent.web.server:app --reload --host 127.0.0.1 --port 8765，并保证 PYTHONPATH 包含 src。

`; + }); +})(); \ No newline at end of file diff --git a/src/agent/web/image/background.jpg b/src/agent/web/image/background.jpg new file mode 100644 index 0000000..8da853a Binary files /dev/null and b/src/agent/web/image/background.jpg differ diff --git a/src/agent/web/image/favicon.svg b/src/agent/web/image/favicon.svg new file mode 100644 index 0000000..6c47d7f --- /dev/null +++ b/src/agent/web/image/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/agent/web/image/microphone.svg b/src/agent/web/image/microphone.svg new file mode 100644 index 0000000..b252186 --- /dev/null +++ b/src/agent/web/image/microphone.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/agent/web/index.html b/src/agent/web/index.html new file mode 100644 index 0000000..38efb5b --- /dev/null +++ b/src/agent/web/index.html @@ -0,0 +1,132 @@ + + + + + + minor · Chatbot + + + + + + + + +
+ + + +
+
+ +
+

minor · Chatbot

+

本地多模态对话（Web）

+
+ +
+
+
+

开始对话

+

输入文字，或上传图片 / 音频 后点击发送。

+

支持用户音频输入、助手文本/语音/双模输出、会话本地媒体持久化。

+
+ +
+
+
+
+
+
+ + +
+
+ + + + +
+
+
+ + +
+
+
+
+
+ + + + + diff --git a/src/agent/web/server.py b/src/agent/web/server.py new file mode 100644 index 0000000..6b830ce --- /dev/null +++ b/src/agent/web/server.py @@ -0,0 +1,406 @@ +""" +为 ``web`` 目录下的静态前端提供 API：会话列表、多模态发送（与 ``chatbot_agent`` 一致）。 + +从仓库 ``Agent`` 目录、且 ``PYTHONPATH`` 包含 ``src`` 时启动:: + + uvicorn agent.web.server:app --reload --host 127.0.0.1 --port 8765 +""" +from __future__ import annotations + +import base64 +import mimetypes +import os +import shutil +import sys +import tempfile +import uuid +from pathlib import Path +from typing import Any + +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse +_SRC = Path(__file__).resolve().parent.parent.parent +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +from agent import chatbot_agent as ca # noqa: E402 +from agent.web import ui_session as ui # noqa: E402 +from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS # noqa: E402 + +_WEB_DIR = Path(__file__).resolve().parent +_IMAGE_DIR = _WEB_DIR / "image" + +_AUDIO_EXT = frozenset({".wav", ".mp3", ".m4a", ".flac", ".ogg", ".webm"}) + + +def _data_url_for_file(fp: str) -> str | None: + """将本地文件读入为 data URL，供前端直接展示（不单独提供文件下载路由）。""" + if not fp or not os.path.isfile(fp): + return None + mime, _ = mimetypes.guess_type(fp) + if not mime: + ext = os.path.splitext(fp)[1].lower() + if ext in IMAGE_FILE_EXTENSIONS: + mime = {".png": "image/png", ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp"}.get( + ext, "image/jpeg" + ) + elif ext in _AUDIO_EXT: + mime = { + ".wav": "audio/wav", + ".mp3": "audio/mpeg", + ".m4a": "audio/mp4", + ".flac": "audio/flac", + ".ogg": "audio/ogg", + }.get(ext, "application/octet-stream") + else: + mime = "application/octet-stream" + try: + with open(fp, "rb") as f: + raw = f.read() + b64 = base64.standard_b64encode(raw).decode("ascii") + except OSError: + return None + return f"data:{mime};base64,{b64}" + + +def _serialize_content(content: Any, *, meta: dict[str, Any] | None = None, role: str | None = None) -> Any: + """将一条消息中的 content 转为 JSON 友好结构；文件以内联 data URL 输出。""" + meta = meta or {} + if isinstance(content, str): + return content + if not isinstance(content, list): + return str(content) + parts: list[dict[str, Any]] = [] + for piece in content: + if isinstance(piece, str) and piece.strip(): + parts.append({"type": "text", "text": piece.strip()}) + continue + if not isinstance(piece, dict): + continue + if piece.get("type") == "text": + parts.append({"type": "text", "text": piece.get("text") or ""}) + continue + fp = piece.get("path") or (piece.get("file") or {}).get("path") + if piece.get("type") == "image_url": + fp = fp or (piece.get("image_url") or {}).get("url") + if piece.get("type") == "audio": + fp = fp or piece.get("audio_path") + if not fp or not os.path.isfile(fp): + continue + data_url = _data_url_for_file(fp) + if not data_url: + continue + ext = os.path.splitext(fp)[1].lower() + if ext in IMAGE_FILE_EXTENSIONS: + parts.append({"type": "image", "url": data_url, "path": fp, "name": os.path.basename(fp)}) + elif ext in _AUDIO_EXT: + audio_meta = meta.get("audio") if isinstance(meta.get("audio"), dict) else {} + parts.append({ + "type": "audio", + "url": data_url, + "path": fp, + "name": os.path.basename(fp), + "autoplay": bool(meta.get("autoplay") or audio_meta.get("autoplay")) if role == "assistant" else False, + "duration_seconds": audio_meta.get("duration_seconds"), + }) + else: + parts.append({"type": "file", "url": data_url, "path": fp, "name": os.path.basename(fp)}) + return parts + + +def _serialize_messages(hist: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + hist = hist or [] + # 找到最后一条 assistant 消息的索引，用于控制自动播放 + last_assistant_idx = -1 + for i in range(len(hist) - 1, -1, -1): + if hist[i].get("role") == "assistant": + last_assistant_idx = i + break + + out = [] + for idx, m in enumerate(hist): + role = m.get("role") + content = m.get("content") + meta = dict(m.get("meta") or {}) + meta.setdefault("output_type", "text" if role != "assistant" else meta.get("output_type") or "text") + if role == "assistant": + meta.setdefault("autoplay", bool((meta.get("audio") or {}).get("autoplay") or meta.get("autoplay"))) + + serialized_content = _serialize_content(content, meta=meta, role=role) + + # ---------- 音频部件处理 ---------- + if role == "assistant" and meta.get("audio"): + audio_meta = meta.get("audio") if isinstance(meta.get("audio"), dict) else {} + # 检查 content 中是否已经包含了音频类型的部件（例如从 manifest 加载时生成的） + already_has_audio = False + if isinstance(serialized_content, list): + for part in serialized_content: + if isinstance(part, dict) and part.get("type") == "audio": + already_has_audio = True + # 修正历史消息的 autoplay 为 False（仅最新一条可允许） + if idx != last_assistant_idx: + part["autoplay"] = False + break + + # 只有 content 中还没有音频部件时，才追加额外的 audio_part + if not already_has_audio: + is_latest = (idx == last_assistant_idx) + autoplay_final = is_latest and (meta.get("autoplay") or audio_meta.get("autoplay")) + audio_part = { + "type": "audio", + "url": _data_url_for_file(audio_meta.get("path")), + "name": "语音回复", + "autoplay": autoplay_final, + "duration_seconds": audio_meta.get("duration_seconds"), + } + if audio_part["url"]: + if isinstance(serialized_content, list): + serialized_content.append(audio_part) + elif serialized_content: + serialized_content = [{"type": "text", "text": serialized_content}, audio_part] + else: + serialized_content = [audio_part] + + out.append({"role": role, "content": serialized_content, "meta": meta}) + return out + + +def _bootstrap() -> tuple[str, list[str], list[dict[str, Any]]]: + ids = ui.list_session_ids_ordered() + if ids: + sid = ids[0] + hist = ui.load_ui_messages_for_session(sid) + else: + sid = ui.new_timestamp_session_id() + hist = [] + ids = [sid] + return sid, ids, hist + + +app = FastAPI(title="minor web chat", version="1.0.0") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/api/bootstrap") +def api_bootstrap() -> dict[str, Any]: + sid, ids, hist = _bootstrap() + return {"sessionId": sid, "sessions": ids, "messages": _serialize_messages(hist)} + + +@app.post("/api/sessions/new") +def api_new_session() -> dict[str, Any]: + sid = ui.new_timestamp_session_id() + ids = ui.list_session_ids_ordered() + ids = [sid] + [x for x in ids if x != sid] + return {"sessionId": sid, "sessions": ids, "messages": []} + + +@app.get("/api/sessions/{session_id}/messages") +def api_session_messages(session_id: str) -> dict[str, Any]: + if not session_id: + raise HTTPException(status_code=400, detail="缺少 session_id") + hist = ui.load_ui_messages_for_session(session_id) + ids = ui.list_session_ids_ordered() + if session_id not in ids: + ids = [session_id, *ids] + return {"sessionId": session_id, "sessions": ids, "messages": _serialize_messages(hist)} + + +@app.post("/api/sessions/clear") +def api_clear_session(session_id: str = Form(...)) -> dict[str, Any]: + if not session_id: + raise HTTPException(status_code=400, detail="缺少 session_id") + messages, _ = ui.clear_ui_and_manifest(session_id) + ids = ui.list_session_ids_ordered() + if session_id not in ids: + ids = [session_id, *ids] + return {"sessionId": session_id, "sessions": ids, "messages": _serialize_messages(messages)} + + +@app.post("/api/sessions/delete") +def api_delete_session(session_id: str = Form(...)) -> dict[str, Any]: + if not session_id: + raise HTTPException(status_code=400, detail="缺少 session_id") + ui.delete_session(session_id) + sid, ids, hist = _bootstrap() + return {"sessionId": sid, "sessions": ids, "messages": _serialize_messages(hist)} + + +def _materialize_uploads_to_temp( + images: list[UploadFile] | None, + audio: UploadFile | None, + text: str, +) -> tuple[dict[str, Any], str]: + """将上传文件落盘到临时目录，返回 ``(query_dict, tmp_root)``；调用方须在 finally 中删除 ``tmp_root``。""" + tmp_root = tempfile.mkdtemp(prefix="web_mm_") + file_paths: list[str] = [] + for uf in images or []: + if uf is None or not uf.filename: + continue + ext = Path(uf.filename).suffix or "" + dest = os.path.join(tmp_root, f"{uuid.uuid4().hex}{ext}") + with open(dest, "wb") as f: + shutil.copyfileobj(uf.file, f) + file_paths.append(dest) + + if audio is not None and audio.filename: + ext = Path(audio.filename).suffix or ".bin" + dest = os.path.join(tmp_root, f"{uuid.uuid4().hex}{ext}") + with open(dest, "wb") as f: + shutil.copyfileobj(audio.file, f) + file_paths.append(dest) + + query = {"text": (text or "").strip(), "files": file_paths} + return query, tmp_root + + +def _chat_start_core(session_id: str, query: dict[str, Any]) -> dict[str, Any]: + """追加 user、复制附件到会话目录并写 manifest（不调用模型）。""" + hist = ui.load_ui_messages_for_session(session_id) + before = len(hist or []) + new_hist, cleared = ui.add_message(hist or [], query) + if cleared is None: + raise HTTPException(status_code=400, detail="请输入文字或上传图片/音频") + if len(new_hist) == before: + raise HTTPException(status_code=400, detail="请输入文字或上传图片/音频") + + ui.rewrite_user_files_for_last_turn(new_hist, session_id) + ui.sync_manifest_from_chat_history(session_id, new_hist, "") + + ids = ui.list_session_ids_ordered() + if session_id not in ids: + ids = [session_id, *ids] + return { + "sessionId": session_id, + "sessions": ids, + "messages": _serialize_messages(new_hist), + } + + +def _chat_complete_core(session_id: str, output_type: str = "text") -> dict[str, Any]: + """对 manifest 中末条 user 执行视觉解析与 ``chain_with_history.invoke``。""" + hist = ui.load_ui_messages_for_session(session_id) + if not hist or hist[-1].get("role") != "user": + raise HTTPException( + status_code=400, + detail="没有待完成的用户消息，请先调用 /api/chat/start", + ) + new_hist = ca.execute_agent([*hist], session_id, output_type=output_type) + ui.prune_memory_session_ids() + ids = ui.list_session_ids_ordered() + if session_id not in ids: + ids = [session_id, *ids] + return { + "sessionId": session_id, + "sessions": ids, + "messages": _serialize_messages(new_hist), + } + + +@app.post("/api/chat/start") +async def api_chat_start( + session_id: str = Form(...), + text: str = Form(""), + images: list[UploadFile] | None = File(None), + audio: UploadFile | None = File(None), +) -> dict[str, Any]: + """ + 仅追加用户消息并持久化到 manifest（含附件落盘）；不调用模型。 + 前端应先调用本接口再调用 ``/api/chat/complete``。 + """ + if not session_id: + raise HTTPException(status_code=400, detail="缺少 session_id") + + query, tmp_root = _materialize_uploads_to_temp(images, audio, text) + try: + return _chat_start_core(session_id, query) + finally: + shutil.rmtree(tmp_root, ignore_errors=True) + + +@app.post("/api/chat/complete") +async def api_chat_complete( + session_id: str = Form(...), + output_type: str = Form("text"), +) -> dict[str, Any]: + """对当前会话最近一次 ``/api/chat/start`` 提交执行模型推理并返回完整消息列表。""" + if not session_id: + raise HTTPException(status_code=400, detail="缺少 session_id") + return _chat_complete_core(session_id, output_type=output_type) + + +@app.post("/api/chat") +async def api_chat( + session_id: str = Form(...), + text: str = Form(""), + images: list[UploadFile] | None = File(None), + audio: UploadFile | None = File(None), + output_type: str = Form("text"), +) -> dict[str, Any]: + """ + 兼容旧客户端：等价于依次 ``/api/chat/start`` + ``/api/chat/complete``（仍阻塞至推理结束）。 + """ + if not session_id: + raise HTTPException(status_code=400, detail="缺少 session_id") + + query, tmp_root = _materialize_uploads_to_temp(images, audio, text) + try: + _chat_start_core(session_id, query) + return _chat_complete_core(session_id, output_type=output_type) + finally: + shutil.rmtree(tmp_root, ignore_errors=True) + + + +# 先注册 API，再挂静态文件，避免 ``/`` 抢路由 +@app.get("/api/health") +def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.get("/favicon.ico") +def favicon() -> FileResponse: + for name, mime in ( + ("favicon.ico", "image/x-icon"), + ("favicon.png", "image/png"), + ("favicon.svg", "image/svg+xml"), + ): + path = _IMAGE_DIR / name + if path.is_file(): + return FileResponse(path, media_type=mime) + raise HTTPException(status_code=404, detail="web/image 目录下缺少 favicon.ico / favicon.png / favicon.svg") + + +@app.get("/image/{asset_name}") +def serve_image_asset(asset_name: str) -> FileResponse: + path = (_IMAGE_DIR / asset_name).resolve() + try: + path.relative_to(_IMAGE_DIR.resolve()) + except ValueError as exc: + raise HTTPException(status_code=404, detail="图片资源不存在") from exc + if not path.is_file(): + raise HTTPException(status_code=404, detail=f"web/image 目录下缺少 {asset_name}") + mime, _ = mimetypes.guess_type(str(path)) + return FileResponse(path, media_type=mime or "application/octet-stream") + + +@app.get("/") +def index_page() -> FileResponse: + return FileResponse(_WEB_DIR / "index.html") + + +@app.get("/app.css") +def serve_css() -> FileResponse: + return FileResponse(_WEB_DIR / "app.css") + + +@app.get("/app.js") +def serve_js() -> FileResponse: + return FileResponse(_WEB_DIR / "app.js") diff --git a/src/agent/web/ui_session.py b/src/agent/web/ui_session.py new file mode 100644 index 0000000..15f98e0 --- /dev/null +++ b/src/agent/web/ui_session.py @@ -0,0 +1,461 @@ +""" +Web/API 与会话 UI 状态：会话 id 列表、manifest、多模态 user 消息结构、附件落盘。 +与 LangChain 链解耦；execute_agent 仍在 chatbot_agent 中，此处提供其依赖的同步与解析函数。 +""" +from __future__ import annotations + +import json +import os +import shutil +import sqlite3 +import uuid +from datetime import datetime + +from agent.utils.agent_utils import ( + CHAT_HISTORY_DB_FS, + MULTI_INPUTS_ROOT, + assistant_text, + extract_result_from_invoke, +) +from agent.utils.image_utils import IMAGE_FILE_EXTENSIONS +from agent.chatbot_agent import get_history + +_AUDIO_EXTENSIONS = {".wav", ".mp3", ".m4a", ".flac", ".ogg"} +_memory_session_ids: set[str] = set() + + +def manifest_path(session_id: str) -> str: + return os.path.join(MULTI_INPUTS_ROOT, session_id, "ui_messages.json") + + +def session_dir(session_id: str) -> str: + return os.path.join(MULTI_INPUTS_ROOT, session_id) + + +def list_session_ids_ordered() -> list[str]: + if not os.path.isfile(CHAT_HISTORY_DB_FS): + return [] + conn = sqlite3.connect(CHAT_HISTORY_DB_FS) + try: + cur = conn.execute( + "SELECT session_id FROM message_store GROUP BY session_id ORDER BY MAX(id) DESC" + ) + return [row[0] for row in cur.fetchall() if row[0]] + finally: + conn.close() + + +def new_timestamp_session_id() -> str: + base = datetime.now().strftime("%Y%m%d_%H%M%S") + existing = set(list_session_ids_ordered()) | _memory_session_ids + if base not in existing: + _memory_session_ids.add(base) + return base + n = 1 + while True: + cand = f"{base}_{n}" + if cand not in existing: + _memory_session_ids.add(cand) + return cand + n += 1 + + +def prune_memory_session_ids() -> None: + persisted = set(list_session_ids_ordered()) + for sid in list(_memory_session_ids): + if sid in persisted: + _memory_session_ids.discard(sid) + + +def delete_session(session_id: str) -> None: + if not session_id: + return + if os.path.isfile(CHAT_HISTORY_DB_FS): + conn = sqlite3.connect(CHAT_HISTORY_DB_FS) + try: + conn.execute("DELETE FROM message_store WHERE session_id = ?", (session_id,)) + conn.commit() + finally: + conn.close() + _memory_session_ids.discard(session_id) + shutil.rmtree(session_dir(session_id), ignore_errors=True) + + +def _lc_content_for_ui_messages(content) -> str | list: + if isinstance(content, str): + return content + if isinstance(content, list): + t = assistant_text(content) + return t if t else str(content) + return str(content) + + +def _load_ui_messages_from_db(session_id: str) -> list: + if not session_id: + return [] + rows: list[dict] = [] + for m in get_history(session_id).messages: + if m.type == "human": + rows.append({"role": "user", "content": _lc_content_for_ui_messages(m.content)}) + elif m.type in ("ai", "assistant"): + text = assistant_text(m.content) or extract_result_from_invoke(m) + rows.append({"role": "assistant", "content": text or "（空回复）"}) + return rows + + +def _build_user_chat_content(query) -> str | list | None: + parts: list = [] + for filepath in query.get("files", []) or []: + if filepath: + parts.append({"path": filepath}) + text = (query.get("text") or "").strip() + if text: + parts.append({"type": "text", "text": text}) + if not parts: + return None + if len(parts) == 1 and isinstance(parts[0], dict) and parts[0].get("type") == "text": + return parts[0]["text"] + return parts + + +def collect_media_paths_and_query_text(user_msg: dict) -> tuple[list[str], list[str], str]: + raw = user_msg.get("content") + text_segments: list[str] = [] + image_paths: list[str] = [] + audio_paths: list[str] = [] + + if isinstance(raw, str): + t = raw.strip() + if t: + text_segments.append(t) + return image_paths, audio_paths, "\n".join(text_segments).strip() + + if not isinstance(raw, list): + return image_paths, audio_paths, "" + + for piece in raw: + if isinstance(piece, str): + t = piece.strip() + if t: + text_segments.append(t) + continue + if not isinstance(piece, dict): + continue + if piece.get("type") == "text": + text_content = (piece.get("text") or "").strip() + if text_content: + text_segments.append(text_content) + continue + file_path = piece.get("path") or (piece.get("file") or {}).get("path") + if piece.get("type") == "image_url": + file_path = file_path or (piece.get("image_url") or {}).get("url") + if piece.get("type") == "audio": + file_path = file_path or piece.get("audio_path") + if not file_path: + continue + ext = os.path.splitext(file_path)[1].lower() + if ext in IMAGE_FILE_EXTENSIONS: + image_paths.append(file_path) + elif ext in _AUDIO_EXTENSIONS: + audio_paths.append(file_path) + else: + text_segments.append(f"[用户上传了附件文件: {os.path.basename(file_path)}]") + + return image_paths, audio_paths, "\n".join(text_segments).strip() + + +def rewrite_user_message_files_to_session_dir(user_msg: dict, session_id: str) -> None: + dest_dir = session_dir(session_id) + os.makedirs(dest_dir, exist_ok=True) + raw = user_msg.get("content") + if not isinstance(raw, list): + return + new_list: list = [] + for piece in raw: + if isinstance(piece, str) or not isinstance(piece, dict): + new_list.append(piece) + continue + if piece.get("type") == "text": + new_list.append(piece) + continue + src = piece.get("path") or (piece.get("file") or {}).get("path") + if piece.get("type") == "image_url": + src = src or (piece.get("image_url") or {}).get("url") + if piece.get("type") == "audio": + src = src or piece.get("audio_path") + if not src or not os.path.isfile(src): + new_list.append(piece) + continue + dest_dir_norm = os.path.normpath(dest_dir) + os.sep + src_norm = os.path.normpath(src) + if src_norm.startswith(dest_dir_norm): + new_list.append(piece) + continue + ext = os.path.splitext(src)[1] or "" + dest = os.path.join(dest_dir, f"{uuid.uuid4().hex}{ext}") + shutil.copy2(src, dest) + new_piece = dict(piece) + new_piece["path"] = dest + if piece.get("type") == "image_url": + new_piece["image_url"] = {"url": dest} + if piece.get("type") == "audio": + new_piece["audio_path"] = dest + new_list.append(new_piece) + user_msg["content"] = new_list + + +def _serialize_user_msg_for_manifest(user_msg: dict) -> dict: + raw = user_msg.get("content") + content_list: list[dict] = [] + if isinstance(raw, str): + if raw.strip(): + content_list.append({"type": "text", "text": raw}) + elif isinstance(raw, list): + for piece in raw: + if isinstance(piece, str) and piece.strip(): + content_list.append({"type": "text", "text": piece.strip()}) + elif isinstance(piece, dict): + if piece.get("type") == "text": + content_list.append({"type": "text", "text": piece.get("text") or ""}) + continue + fp = piece.get("path") or (piece.get("file") or {}).get("path") + if piece.get("type") == "image_url": + fp = fp or (piece.get("image_url") or {}).get("url") + if piece.get("type") == "audio": + fp = fp or piece.get("audio_path") + if not fp: + continue + ext = os.path.splitext(fp)[1].lower() + try: + rel = os.path.relpath(os.path.normpath(fp), os.path.normpath(MULTI_INPUTS_ROOT)) + except ValueError: + rel = fp.replace("\\", "/") + if ext in IMAGE_FILE_EXTENSIONS: + part_type = "image_url" + elif ext in _AUDIO_EXTENSIONS: + part_type = "audio" + else: + part_type = "file" + content_list.append({"type": part_type, "relpath": rel.replace("\\", "/")}) + # 不再写入 image_description 字段 + return {"content": content_list} + + +def _manifest_user_entry_to_content(user_entry: dict) -> str | list: + root_norm = os.path.normpath(MULTI_INPUTS_ROOT) + parts_out: list = [] + any_image_missing = False + had_image_parts = False + + # 新格式直接取 "content" + content_data = user_entry.get("content") + if content_data is None: + return "" + + for p in content_data: + part_type = p.get("type") + if part_type == "text": + t = (p.get("text") or "").strip() + if t: + parts_out.append({"type": "text", "text": t}) + continue + rel = (p.get("relpath") or "").replace("/", os.sep) + abs_p = os.path.normpath(os.path.join(root_norm, rel)) + try: + under = os.path.commonpath([abs_p, root_norm]) == root_norm + except ValueError: + under = False + if not under: + if part_type == "image_url": + any_image_missing = True + had_image_parts = True + continue + if part_type == "image_url": + had_image_parts = True + if os.path.isfile(abs_p): + parts_out.append({"type": "image_url", "image_url": {"url": abs_p}, "path": abs_p}) + else: + any_image_missing = True + elif part_type == "audio": + if os.path.isfile(abs_p): + parts_out.append({"type": "audio", "audio_path": abs_p, "path": abs_p}) + else: + parts_out.append({"type": "text", "text": f"[语音已丢失: {os.path.basename(rel) or rel}]"}) + elif part_type == "file": + if os.path.isfile(abs_p): + parts_out.append({"path": abs_p}) + else: + parts_out.append({"type": "text", "text": f"[附件已丢失: {os.path.basename(rel) or rel}]"}) + + # 不再处理 image_description（新格式中没有） + if had_image_parts and any_image_missing: + parts_out.append({"type": "text", "text": "[图片已丢失]"}) + + if not parts_out: + return "" + if len(parts_out) == 1 and parts_out[0].get("type") == "text": + return parts_out[0]["text"] + return parts_out + + +def _serialize_assistant_entry(assistant_msg: dict | None) -> dict: + ac = (assistant_msg or {}).get("content", "") + meta = dict((assistant_msg or {}).get("meta") or {}) + if isinstance(ac, str): + atext = ac + else: + atext = assistant_text(ac) or extract_result_from_invoke(assistant_msg) or "（空回复）" + audio_relpath = None + audio = meta.get("audio") + if isinstance(audio, dict) and audio.get("path"): + try: + audio_relpath = os.path.relpath( + os.path.normpath(audio["path"]), os.path.normpath(MULTI_INPUTS_ROOT) + ) + except ValueError: + audio_relpath = str(audio["path"]).replace("\\", "/") + return { + "text": atext, + "output_type": meta.get("output_type") or "text", + "autoplay": bool(meta.get("autoplay") or (audio or {}).get("autoplay")), + "audio_duration_seconds": (audio or {}).get("duration_seconds") if isinstance(audio, dict) else None, + "audio_relpath": audio_relpath.replace("\\", "/") if audio_relpath else None, + } + + +def sync_manifest_from_chat_history(session_id: str, chat_history: list, latest_user_image_description: str) -> None: + if not session_id: + return + mp = manifest_path(session_id) + turns_out: list[dict] = [] + ch = chat_history or [] + i = 0 + while i < len(ch): + if ch[i].get("role") != "user": + i += 1 + continue + user_msg = ch[i] + assistant_msg = ch[i + 1] if i + 1 < len(ch) and ch[i + 1].get("role") == "assistant" else None + + turns_out.append( + { + "user": _serialize_user_msg_for_manifest(user_msg), + "assistant": _serialize_assistant_entry(assistant_msg), + } + ) + i += 2 if assistant_msg else 1 + + os.makedirs(os.path.dirname(mp), exist_ok=True) + with open(mp, "w", encoding="utf-8") as f: + json.dump({"version": 2, "turns": turns_out}, f, ensure_ascii=False, indent=2) + +def _assistant_manifest_entry_to_content(entry: dict) -> str | list: + text = (entry.get("text") or "").strip() + output_type = entry.get("output_type") or "text" + autoplay = bool(entry.get("autoplay")) + audio_duration_seconds = entry.get("audio_duration_seconds") + audio_relpath = entry.get("audio_relpath") + audio_abs_path = None + if audio_relpath: + root_norm = os.path.normpath(MULTI_INPUTS_ROOT) + audio_abs_path = os.path.normpath(os.path.join(root_norm, audio_relpath.replace("/", os.sep))) + try: + under = os.path.commonpath([audio_abs_path, root_norm]) == root_norm + except ValueError: + under = False + if not under or not os.path.isfile(audio_abs_path): + audio_abs_path = None + parts: list[dict] = [] + if output_type in ("text", "both") and text: + parts.append({"type": "text", "text": text}) + if output_type in ("audio", "both") and audio_abs_path: + parts.append({ + "type": "audio", + "audio_path": audio_abs_path, + "path": audio_abs_path, + "autoplay": autoplay, + "duration_seconds": audio_duration_seconds, + }) + if not parts: + return text + if len(parts) == 1 and parts[0].get("type") == "text": + return parts[0]["text"] + return parts + + +def _load_ui_messages_from_manifest(session_id: str) -> list | None: + mp = manifest_path(session_id) + if not os.path.isfile(mp): + return None + try: + with open(mp, encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError): + return None + if "turns" not in data: + return None + turns = data.get("turns") or [] + if turns == []: + return [] + out: list[dict] = [] + n = len(turns) + for idx, t in enumerate(turns): + out.append({"role": "user", "content": _manifest_user_entry_to_content(t.get("user") or {})}) + asst_entry = t.get("assistant") or {} + if not isinstance(asst_entry, dict): + asst_entry = {"text": str(asst_entry)} + asst_content = _assistant_manifest_entry_to_content(asst_entry) + if idx == n - 1 and not str(asst_entry.get("text") or "").strip() and asst_content == "": + continue + + # 从扁平存储重建与 execute_agent 产物相同的 meta 字典 + meta = { + "output_type": asst_entry.get("output_type", "text"), + "autoplay": asst_entry.get("autoplay", False), + } + audio_relpath = asst_entry.get("audio_relpath") + if audio_relpath: + root_norm = os.path.normpath(MULTI_INPUTS_ROOT) + audio_path = os.path.normpath(os.path.join(root_norm, audio_relpath.replace("/", os.sep))) + if os.path.isfile(audio_path): + meta["audio"] = { + "path": audio_path, + "autoplay": asst_entry.get("autoplay", False), + "duration_seconds": asst_entry.get("audio_duration_seconds"), + "format": "wav", + "sample_rate": 24000, + } + out.append({"role": "assistant", "content": asst_content, "meta": meta}) + return out + + +def load_ui_messages_for_session(session_id: str) -> list: + if not session_id: + return [] + m = _load_ui_messages_from_manifest(session_id) + if m is not None: + return m + return _load_ui_messages_from_db(session_id) + + +def rewrite_user_files_for_last_turn(chat_history: list, session_id: str) -> None: + if session_id and chat_history and chat_history[-1].get("role") == "user": + rewrite_user_message_files_to_session_dir(chat_history[-1], session_id) + + +def add_message(chat_history, query): + chat_history = chat_history or [] + user_content = _build_user_chat_content(query) + if user_content is None: + return chat_history, None + chat_history.append({"role": "user", "content": user_content}) + return chat_history, {"text": "", "files": []} + + +def clear_ui_and_manifest(session_id: str): + if session_id: + mp = manifest_path(session_id) + os.makedirs(os.path.dirname(mp), exist_ok=True) + with open(mp, "w", encoding="utf-8") as f: + json.dump({"version": 1, "turns": []}, f, ensure_ascii=False, indent=2) + return [], {"text": "", "files": []}