diff --git a/src/agent/core/agent_config.json b/src/agent/core/agent_config.json index f7eb8f7..ec25f34 100644 --- a/src/agent/core/agent_config.json +++ b/src/agent/core/agent_config.json @@ -5,12 +5,8 @@ "max_iterations": 200, "llm_model_id": "b53afakf", "tools": [ - "human_interaction", - "directory_listing", - "web_search_class", - "date_query_class", - "doc_tool", - "todo_list" + "browser_control", + "gui_tool(test)" ], "enabled": true, "role": "main", diff --git a/src/agent/core/tool_config.json b/src/agent/core/tool_config.json index 4312e0b..4250411 100644 --- a/src/agent/core/tool_config.json +++ b/src/agent/core/tool_config.json @@ -112,5 +112,13 @@ "permission": "direct", "auto_execute_rules": [], "enabled": true + }, + { + "name": "gui_tool(test)", + "display_name": "GUIToolTest", + "description": "GUI 自动化工具（测试版）。通过截图视觉定位桌面元素，将坐标映射到真实屏幕，并执行 pyautogui 动作。接收一个有序的动作列表，按顺序依次执行。每个动作项支持的操作：click（点击）、type（输入文本，可通过 element_description 指定目标输入框自动聚焦）、drag_and_drop（拖拽）、scroll（滚动）、hotkey（快捷键）、hold_and_press（", + "permission": "direct", + "auto_execute_rules": [], + "enabled": true } ] \ No newline at end of file diff --git a/src/agent/history/data_base/chat_history.db b/src/agent/history/data_base/chat_history.db index 7879be9..1b45d38 100644 Binary files a/src/agent/history/data_base/chat_history.db and b/src/agent/history/data_base/chat_history.db differ diff --git a/src/agent/history/multi_inputs/20260705_092727/ui_messages.json b/src/agent/history/multi_inputs/20260705_092727/ui_messages.json new file mode 100644 index 0000000..066f2d1 --- /dev/null +++ b/src/agent/history/multi_inputs/20260705_092727/ui_messages.json @@ -0,0 +1,16 @@ +{ + "version": 3, + "messages": [ + { + "role": "user", + "user": { + "content": [ + { + "type": "text", + "text": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\r\n2、点击采购管理\r\n3、点击新增供应商\r\n4、在新增供应商页面进行以下操作：\r\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260705_094025/ui_messages.json b/src/agent/history/multi_inputs/20260705_094025/ui_messages.json new file mode 100644 index 0000000..245b8be --- /dev/null +++ b/src/agent/history/multi_inputs/20260705_094025/ui_messages.json @@ -0,0 +1,16 @@ +{ + "version": 3, + "messages": [ + { + "role": "user", + "user": { + "content": [ + { + "type": "text", + "text": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\r\\n2、点击采购管理\\r\\n3、点击新增供应商\\r\\n4、在新增供应商页面进行以下操作：\\r\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260705_095248/ui_messages.json b/src/agent/history/multi_inputs/20260705_095248/ui_messages.json new file mode 100644 index 0000000..f840c1a --- /dev/null +++ b/src/agent/history/multi_inputs/20260705_095248/ui_messages.json @@ -0,0 +1,28 @@ +{ + "version": 3, + "messages": [ + { + "role": "user", + "user": { + "content": [ + { + "type": "text", + "text": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\\\r\\\\n2、点击采购管理\\\\r\\\\n3、点击新增供应商\\\\r\\\\n4、在新增供应商页面进行以下操作：\\\\r\\\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + } + ] + } + }, + { + "role": "assistant", + "assistant": { + "text": "1. 分析上一轮执行结果：上一步调用了gui_tool点击“确定”按钮，截图显示提示框已关闭，当前处于新增供应商页面，所有字段均已正确填写并保存成功。\n2. 任务目标达成：\n - 供应商名称：\"minor\" (已填)\n - 供应商类别：\"工程材料供应商\" (已填)\n - 部门：\"采购部门B\" (已填，且已修正并保存)\n - 职务：\"CEO\" (已填)\n3. 下一步操作：任务已完成，直接回复用户。\n\n任务已完成。已成功新增供应商，信息如下：\n- 供应商名称：minor\n- 供应商类别：工程材料供应商\n- 部门：采购部门B\n- 职务：CEO\n系统提示“保存数据成功”。", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null, + "pending_action": null, + "agent_mode": null + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260705_132839/ui_messages.json b/src/agent/history/multi_inputs/20260705_132839/ui_messages.json new file mode 100644 index 0000000..ec9b617 --- /dev/null +++ b/src/agent/history/multi_inputs/20260705_132839/ui_messages.json @@ -0,0 +1,28 @@ +{ + "version": 3, + "messages": [ + { + "role": "user", + "user": { + "content": [ + { + "type": "text", + "text": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\\\r\\\\n2、点击采购管理\\\\r\\\\n3、点击新增供应商\\\\r\\\\n4、在新增供应商页面进行以下操作：\\\\r\\\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + } + ] + } + }, + { + "role": "assistant", + "assistant": { + "text": "1. 先分析上一轮执行结果：上一个调用了 gui_tool(test) 在“职务”输入框中输入了“CEO”。从截图来看，“职务”输入框中已经成功显示了“CEO”，输入操作成功。\n2. 回顾任务要求，需要输入的信息包括：\n - 供应商名称：“minor” -> 已完成\n - 供应商类别：“工程材料供应商” -> 截图显示默认值即为“工程材料供应商”，符合要求。\n - 部门：“采购部门B” -> 已完成\n - 职务：“CEO” -> 已完成\n3. 所有要求的字段均已按要求填入。任务基本完成（虽然通常接下来会点击保存，但任务描述中只要求输入这些操作）。\n4. 既然所有指定的输入操作都已完成，我可以向用户报告任务完成。\n\n任务已完成。已在新增供应商页面成功输入：\n- 供应商名称：minor\n- 供应商类别：工程材料供应商\n- 部门：采购部门B\n- 职务：CEO", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null, + "pending_action": null, + "agent_mode": null + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/multi_inputs/20260705_135932/ui_messages.json b/src/agent/history/multi_inputs/20260705_135932/ui_messages.json new file mode 100644 index 0000000..60ea011 --- /dev/null +++ b/src/agent/history/multi_inputs/20260705_135932/ui_messages.json @@ -0,0 +1,28 @@ +{ + "version": 3, + "messages": [ + { + "role": "user", + "user": { + "content": [ + { + "type": "text", + "text": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\\\r\\\\n2、点击采购管理\\\\r\\\\n3、点击新增供应商\\\\r\\\\n4、在新增供应商页面进行以下操作：\\\\r\\\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + } + ] + } + }, + { + "role": "assistant", + "assistant": { + "text": "1. **反思**：\n - 上一步我调用了 `gui_tool` 的 `type` 动作，针对“部门”输入框，设置了 `overwrite=True` 并输入“采购部门B”。\n - 查看最新的截图，部门输入框中现在正确显示为“采购部门B”。\n - 回顾任务要求：\n 1. 打开 https://erpdemo.fanpusoft.com/main.aspx# （已完成）\n 2. 点击采购管理 （已完成）\n 3. 点击新增供应商 （已完成）\n 4. 输入供应商名称：“minor” （已完成）\n 5. 输入供应商类别：“工程材料供应商” （已完成，截图中可见）\n 6. 输入部门：“采购部门B” （已完成，刚刚修正）\n 7. 输入职务：“CEO” （已完成，截图中可见）\n - 所有要求的字段都已正确填写。任务已完成。\n\n2. **下一步操作**：任务已完成，直接回复用户。\n\n任务已完成。已在“新增供应商”页面填写以下信息：\n- 供应商名称：minor\n- 供应商类别：工程材料供应商\n- 部门：采购部门B\n- 职务：CEO", + "output_type": "text", + "autoplay": false, + "audio_duration_seconds": null, + "audio_relpath": null, + "pending_action": null, + "agent_mode": null + } + } + ] +} \ No newline at end of file diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_browser_control_0_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_browser_control_0_0.png new file mode 100644 index 0000000..eea3ea7 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_browser_control_0_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_0_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_0_0.png new file mode 100644 index 0000000..6157d93 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_0_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_10_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_10_0.png new file mode 100644 index 0000000..5e5ac8c Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_10_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_11_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_11_0.png new file mode 100644 index 0000000..6707120 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_11_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_12_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_12_0.png new file mode 100644 index 0000000..3461667 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_12_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_1_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_1_0.png new file mode 100644 index 0000000..e9834f9 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_1_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_2_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_2_0.png new file mode 100644 index 0000000..954cee0 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_2_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_3_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_3_0.png new file mode 100644 index 0000000..7e62958 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_3_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_4_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_4_0.png new file mode 100644 index 0000000..a9dbde4 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_4_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_5_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_5_0.png new file mode 100644 index 0000000..2036f0b Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_5_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_6_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_6_0.png new file mode 100644 index 0000000..a1bcb7c Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_6_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_7_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_7_0.png new file mode 100644 index 0000000..a2085e4 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_7_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_8_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_8_0.png new file mode 100644 index 0000000..fe730ce Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_8_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_9_0.png b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_9_0.png new file mode 100644 index 0000000..6712199 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_095248/20260705_095332/_artifact_gui_tool_9_0.png differ diff --git a/src/agent/history/tool_calling/20260705_095248/turn_20260705_095332.json b/src/agent/history/tool_calling/20260705_095248/turn_20260705_095332.json new file mode 100644 index 0000000..16cee9b --- /dev/null +++ b/src/agent/history/tool_calling/20260705_095248/turn_20260705_095332.json @@ -0,0 +1,252 @@ +{ + "session_id": "20260705_095248", + "turn_id": "20260705_095332", + "user_message": { + "role": "user", + "content": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\\\r\\\\n2、点击采购管理\\\\r\\\\n3、点击新增供应商\\\\r\\\\n4、在新增供应商页面进行以下操作：\\\\r\\\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + }, + "tool_calls": [ + { + "name": "browser_control", + "args": { + "action": "open", + "url": "https://erpdemo.fanpusoft.com/main.aspx#\\" + }, + "result_text": "已尝试打开 URL: https://erpdemo.fanpusoft.com/main.aspx#\\", + "result_files": [ + "_artifact_browser_control_0_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "采购管理菜单", + "target_type": "link" + } + ] + }, + "result_text": "1. Clicked target. grounded (399,126)", + "result_files": [ + "_artifact_gui_tool_0_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "供应商管理菜单", + "target_type": "list_item" + } + ] + }, + "result_text": "1. Clicked target. grounded (44,355)", + "result_files": [ + "_artifact_gui_tool_1_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "新增供应商按钮", + "target_type": "link" + } + ] + }, + "result_text": "1. Clicked target. grounded (323,311)", + "result_files": [ + "_artifact_gui_tool_2_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "type", + "element_description": "供应商名称输入框", + "text": "minor" + } + ] + }, + "result_text": "1. Clicked target and typed text. grounded (309,457)", + "result_files": [ + "_artifact_gui_tool_3_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "type", + "element_description": "部门输入框", + "text": "采购部门B" + }, + { + "action": "type", + "element_description": "职务输入框", + "text": "CEO" + } + ] + }, + "result_text": "1. Clicked target and typed text. grounded (308,560)\n2. Clicked target and typed text. grounded (460,566)", + "result_files": [ + "_artifact_gui_tool_4_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "hotkey", + "keys": [ + "enter" + ] + } + ] + }, + "result_text": "1. Pressed hotkey ['enter'].", + "result_files": [ + "_artifact_gui_tool_5_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "保存按钮", + "target_type": "button" + } + ] + }, + "result_text": "1. Clicked target. grounded (176,234)", + "result_files": [ + "_artifact_gui_tool_6_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "部门输入框", + "target_type": "input" + } + ] + }, + "result_text": "1. Clicked target. grounded (299,565)", + "result_files": [ + "_artifact_gui_tool_7_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "确定按钮", + "target_type": "button" + } + ] + }, + "result_text": "1. Clicked target. grounded (572,640)", + "result_files": [ + "_artifact_gui_tool_8_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "部门输入框", + "target_type": "input" + } + ] + }, + "result_text": "1. Clicked target. grounded (302,563)", + "result_files": [ + "_artifact_gui_tool_9_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "type", + "element_description": "部门输入框", + "text": "采购部门B", + "overwrite": true + } + ] + }, + "result_text": "1. Clicked target and typed text. grounded (304,562)", + "result_files": [ + "_artifact_gui_tool_10_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "保存按钮", + "target_type": "button" + } + ] + }, + "result_text": "1. Clicked target. grounded (176,235)", + "result_files": [ + "_artifact_gui_tool_11_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool", + "args": { + "actions": [ + { + "action": "click", + "element_description": "确定按钮", + "target_type": "button" + } + ] + }, + "result_text": "1. Clicked target. grounded (571,646)", + "result_files": [ + "_artifact_gui_tool_12_0.png" + ], + "result_images": [] + } + ], + "assistant_response": "1. 分析上一轮执行结果：上一步调用了gui_tool点击“确定”按钮，截图显示提示框已关闭，当前处于新增供应商页面，所有字段均已正确填写并保存成功。\n2. 任务目标达成：\n - 供应商名称：\"minor\" (已填)\n - 供应商类别：\"工程材料供应商\" (已填)\n - 部门：\"采购部门B\" (已填，且已修正并保存)\n - 职务：\"CEO\" (已填)\n3. 下一步操作：任务已完成，直接回复用户。\n\n任务已完成。已成功新增供应商，信息如下：\n- 供应商名称：minor\n- 供应商类别：工程材料供应商\n- 部门：采购部门B\n- 职务：CEO\n系统提示“保存数据成功”。", + "timestamp": "20260705_095332" +} \ No newline at end of file diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_browser_control_0_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_browser_control_0_0.png new file mode 100644 index 0000000..8808841 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_browser_control_0_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_0_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_0_0.png new file mode 100644 index 0000000..af2d3eb Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_0_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_1_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_1_0.png new file mode 100644 index 0000000..b07e80b Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_1_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_2_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_2_0.png new file mode 100644 index 0000000..1409c74 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_2_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_3_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_3_0.png new file mode 100644 index 0000000..8d6da17 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_3_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_4_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_4_0.png new file mode 100644 index 0000000..d11d3a5 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_4_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_5_0.png b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_5_0.png new file mode 100644 index 0000000..f60d6b0 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_132839/20260705_132843/_artifact_gui_tool(test)_5_0.png differ diff --git a/src/agent/history/tool_calling/20260705_132839/turn_20260705_132843.json b/src/agent/history/tool_calling/20260705_132839/turn_20260705_132843.json new file mode 100644 index 0000000..41536b8 --- /dev/null +++ b/src/agent/history/tool_calling/20260705_132839/turn_20260705_132843.json @@ -0,0 +1,131 @@ +{ + "session_id": "20260705_132839", + "turn_id": "20260705_132843", + "user_message": { + "role": "user", + "content": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\\\r\\\\n2、点击采购管理\\\\r\\\\n3、点击新增供应商\\\\r\\\\n4、在新增供应商页面进行以下操作：\\\\r\\\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + }, + "tool_calls": [ + { + "name": "browser_control", + "args": { + "action": "open", + "url": "https://erpdemo.fanpusoft.com/main.aspx" + }, + "result_text": "已尝试打开 URL: https://erpdemo.fanpusoft.com/main.aspx", + "result_files": [ + "_artifact_browser_control_0_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "click", + "element_description": "顶部导航栏中的采购管理菜单", + "target_type": "button" + } + ] + }, + "result_text": "1. Clicked target at (791,136).", + "result_files": [ + "_artifact_gui_tool(test)_0_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "click", + "element_description": "新增供应商按钮", + "target_type": "button" + } + ] + }, + "result_text": "1. Clicked target at (636,319).", + "result_files": [ + "_artifact_gui_tool(test)_1_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "type", + "element_description": "供应商名称输入框", + "text": "minor" + } + ] + }, + "result_text": "1. Clicked target and typed text at (547,496).", + "result_files": [ + "_artifact_gui_tool(test)_2_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "hotkey", + "keys": [ + "enter" + ] + }, + { + "action": "wait", + "wait_seconds": 0.5 + } + ] + }, + "result_text": "1. Pressed hotkey ['enter'].\n2. Waited 0.50 seconds.", + "result_files": [ + "_artifact_gui_tool(test)_3_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "type", + "element_description": "部门输入框", + "text": "采购部门B" + } + ] + }, + "result_text": "1. Clicked target and typed text at (493,603).", + "result_files": [ + "_artifact_gui_tool(test)_4_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "type", + "element_description": "职务输入框", + "text": "CEO" + } + ] + }, + "result_text": "1. Clicked target and typed text at (887,611).", + "result_files": [ + "_artifact_gui_tool(test)_5_0.png" + ], + "result_images": [] + } + ], + "assistant_response": "1. 先分析上一轮执行结果：上一个调用了 gui_tool(test) 在“职务”输入框中输入了“CEO”。从截图来看，“职务”输入框中已经成功显示了“CEO”，输入操作成功。\n2. 回顾任务要求，需要输入的信息包括：\n - 供应商名称：“minor” -> 已完成\n - 供应商类别：“工程材料供应商” -> 截图显示默认值即为“工程材料供应商”，符合要求。\n - 部门：“采购部门B” -> 已完成\n - 职务：“CEO” -> 已完成\n3. 所有要求的字段均已按要求填入。任务基本完成（虽然通常接下来会点击保存，但任务描述中只要求输入这些操作）。\n4. 既然所有指定的输入操作都已完成，我可以向用户报告任务完成。\n\n任务已完成。已在新增供应商页面成功输入：\n- 供应商名称：minor\n- 供应商类别：工程材料供应商\n- 部门：采购部门B\n- 职务：CEO", + "timestamp": "20260705_132843" +} \ No newline at end of file diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_browser_control_0_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_browser_control_0_0.png new file mode 100644 index 0000000..f5f978f Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_browser_control_0_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_0_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_0_0.png new file mode 100644 index 0000000..e468dee Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_0_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_1_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_1_0.png new file mode 100644 index 0000000..67aed15 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_1_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_2_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_2_0.png new file mode 100644 index 0000000..8cfbd97 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_2_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_3_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_3_0.png new file mode 100644 index 0000000..771cdb8 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_3_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_4_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_4_0.png new file mode 100644 index 0000000..0eba07f Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_4_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_5_0.png b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_5_0.png new file mode 100644 index 0000000..c274c85 Binary files /dev/null and b/src/agent/history/tool_calling/20260705_135932/20260705_135934/_artifact_gui_tool(test)_5_0.png differ diff --git a/src/agent/history/tool_calling/20260705_135932/turn_20260705_135934.json b/src/agent/history/tool_calling/20260705_135932/turn_20260705_135934.json new file mode 100644 index 0000000..fb84268 --- /dev/null +++ b/src/agent/history/tool_calling/20260705_135932/turn_20260705_135934.json @@ -0,0 +1,139 @@ +{ + "session_id": "20260705_135932", + "turn_id": "20260705_135934", + "user_message": { + "role": "user", + "content": "1、打开https://erpdemo.fanpusoft.com/main.aspx#\\\\r\\\\n2、点击采购管理\\\\r\\\\n3、点击新增供应商\\\\r\\\\n4、在新增供应商页面进行以下操作：\\\\r\\\\n输入供应商名称：“minor”、输入供应商类别：“工程材料供应商”、输入部门：“采购部门B”、输入职务：“CEO”" + }, + "tool_calls": [ + { + "name": "browser_control", + "args": { + "action": "open", + "url": "https://erpdemo.fanpusoft.com/main.aspx#\\" + }, + "result_text": "已尝试打开 URL: https://erpdemo.fanpusoft.com/main.aspx#\\", + "result_files": [ + "_artifact_browser_control_0_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "click", + "element_description": "采购管理图标" + } + ] + }, + "result_text": "1. Clicked target at (797,119).", + "result_files": [ + "_artifact_gui_tool(test)_0_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "click", + "element_description": "新增供应商按钮" + } + ] + }, + "result_text": "1. Clicked target at (655,326).", + "result_files": [ + "_artifact_gui_tool(test)_1_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "type", + "element_description": "供应商名称输入框", + "text": "minor" + }, + { + "action": "type", + "element_description": "部门输入框", + "text": "采购部门B" + }, + { + "action": "type", + "element_description": "职务输入框", + "text": "CEO" + } + ] + }, + "result_text": "1. Clicked target and typed text at (547,498).\n2. Clicked target and typed text at (543,608).\n3. Clicked target and typed text at (912,610).", + "result_files": [ + "_artifact_gui_tool(test)_2_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "click", + "text": "CEO", + "element_description": "职务输入框弹出列表中的CEO选项" + } + ] + }, + "result_text": "1. Clicked target at (889,653).", + "result_files": [ + "_artifact_gui_tool(test)_3_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "click", + "element_description": "部门输入框", + "overwrite": true + }, + { + "action": "type", + "text": "采购部门B" + } + ] + }, + "result_text": "1. Clicked target at (547,605).\n2. Typed text.", + "result_files": [ + "_artifact_gui_tool(test)_4_0.png" + ], + "result_images": [] + }, + { + "name": "gui_tool(test)", + "args": { + "actions": [ + { + "action": "type", + "element_description": "部门输入框", + "text": "采购部门B", + "overwrite": true + } + ] + }, + "result_text": "1. Clicked target and typed text at (616,607).", + "result_files": [ + "_artifact_gui_tool(test)_5_0.png" + ], + "result_images": [] + } + ], + "assistant_response": "1. **反思**：\n - 上一步我调用了 `gui_tool` 的 `type` 动作，针对“部门”输入框，设置了 `overwrite=True` 并输入“采购部门B”。\n - 查看最新的截图，部门输入框中现在正确显示为“采购部门B”。\n - 回顾任务要求：\n 1. 打开 https://erpdemo.fanpusoft.com/main.aspx# （已完成）\n 2. 点击采购管理 （已完成）\n 3. 点击新增供应商 （已完成）\n 4. 输入供应商名称：“minor” （已完成）\n 5. 输入供应商类别：“工程材料供应商” （已完成，截图中可见）\n 6. 输入部门：“采购部门B” （已完成，刚刚修正）\n 7. 输入职务：“CEO” （已完成，截图中可见）\n - 所有要求的字段都已正确填写。任务已完成。\n\n2. **下一步操作**：任务已完成，直接回复用户。\n\n任务已完成。已在“新增供应商”页面填写以下信息：\n- 供应商名称：minor\n- 供应商类别：工程材料供应商\n- 部门：采购部门B\n- 职务：CEO", + "timestamp": "20260705_135934" +} \ No newline at end of file diff --git a/src/agent/tools/__init__.py b/src/agent/tools/__init__.py index ef6a1e6..eddfa22 100644 --- a/src/agent/tools/__init__.py +++ b/src/agent/tools/__init__.py @@ -4,7 +4,6 @@ from .gui_tool import GUITool from .tool_rag import RAGTool from .tool_agent_call import CallSubAgentTool from .doc_tool import DocTool - _all_tools = [ WebSearch(), DateQuery(), diff --git a/src/agent/tools/__pycache__/__init__.cpython-312.pyc b/src/agent/tools/__pycache__/__init__.cpython-312.pyc index 4c0f7fa..b63efeb 100644 Binary files a/src/agent/tools/__pycache__/__init__.cpython-312.pyc and b/src/agent/tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/gui_tool.cpython-312.pyc b/src/agent/tools/__pycache__/gui_tool.cpython-312.pyc index 1318132..8685009 100644 Binary files a/src/agent/tools/__pycache__/gui_tool.cpython-312.pyc and b/src/agent/tools/__pycache__/gui_tool.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/gui_tool_test.cpython-312.pyc b/src/agent/tools/__pycache__/gui_tool_test.cpython-312.pyc new file mode 100644 index 0000000..c54b724 Binary files /dev/null and b/src/agent/tools/__pycache__/gui_tool_test.cpython-312.pyc differ diff --git a/src/agent/tools/__pycache__/tool.cpython-312.pyc b/src/agent/tools/__pycache__/tool.cpython-312.pyc index c361df5..46a86f6 100644 Binary files a/src/agent/tools/__pycache__/tool.cpython-312.pyc and b/src/agent/tools/__pycache__/tool.cpython-312.pyc differ diff --git a/src/agent/tools/gui_tool.py b/src/agent/tools/gui_tool.py index 32d3302..a5f52ad 100644 --- a/src/agent/tools/gui_tool.py +++ b/src/agent/tools/gui_tool.py @@ -1,3 +1,10 @@ +""" +GUI 自动化工具。 +采用批量坐标生成方案：一次截图 + 一次 LLM 调用批量生成所有动作坐标。 +N 个需要定位的动作 = 1 次 LLM 调用。 +提示词中内置 One-shot 示例纠正大模型输出格式。 +""" + from __future__ import annotations import asyncio @@ -6,7 +13,7 @@ import io import re import sys import time -from typing import Any, Literal, Optional +from typing import Any, Literal, Optional, Tuple import pyautogui from langchain.tools import BaseTool @@ -17,17 +24,13 @@ from pydantic import BaseModel, Field from agent.core.llm import llm import agent.utils.env_utils as env_utils -# GUI 模型实例缓存（None 表示使用主模型 llm） +# ---------- GUI 模型实例缓存 ---------- _gui_llm = None _gui_llm_initialized = False def _get_gui_llm(): - """获取 GUI 工具实际使用的 LLM 实例。 - - 优先使用 gui_config.json 中配置的 GUI 专用模型， - 若未配置则回退到 agent 主模型 llm。 - """ + """获取 GUI 工具实际使用的 LLM 实例（优先 GUI 专用模型，否则回退主模型）。""" global _gui_llm, _gui_llm_initialized if not _gui_llm_initialized: _gui_llm_initialized = True @@ -38,6 +41,8 @@ def _get_gui_llm(): _gui_llm = None return _gui_llm if _gui_llm is not None else llm + +# ---------- 类型定义 ---------- GUIAction = Literal[ "click", "type", @@ -54,47 +59,41 @@ GUITargetType = Literal["auto", "button", "input", "checkbox", "list_item", "lin class GUIActionItem(BaseModel): - """A single GUI action within an ordered sequence.""" - action: GUIAction = Field( - ..., - description="GUI action to execute.", - ) + """单个 GUI 动作。""" + action: GUIAction = Field(..., description="GUI 操作类型。") element_description: Optional[str] = Field( default=None, - description="Natural-language description of the target screen element for grounded actions (click, type, scroll, drag_and_drop, move_to). For type action, providing this will first locate and click the target input field before typing.", + description="目标元素的自然语言描述。对于 type 操作，提供此项会先定位并点击目标输入框再输入。", ) ending_description: Optional[str] = Field( default=None, - description="Natural-language description of the drag end target.", + description="拖拽终点的自然语言描述（仅 drag_and_drop 使用）。", ) - text: str = Field(default="", description="Text to type.") - overwrite: bool = Field(default=False, description="Select all existing text before typing.") - enter: bool = Field(default=False, description="Press Enter after typing.") - num_clicks: int = Field(default=1, description="Number of mouse clicks.") - button_type: Literal["left", "middle", "right"] = Field(default="left", description="Mouse button.") - hold_keys: list[str] = Field(default_factory=list, description="Modifier keys to hold while clicking or dragging.") - keys: list[str] = Field(default_factory=list, description="Hotkey keys or keys to press.") - press_keys: list[str] = Field(default_factory=list, description="Keys to press while holding hold_keys.") - clicks: int = Field(default=0, description="Scroll click amount. Negative means down.") - shift: bool = Field(default=False, description="Use horizontal scrolling when true.") - wait_seconds: float = Field(default=0.5, description="Seconds to wait.") + text: str = Field(default="", description="要输入的文本。") + overwrite: bool = Field(default=False, description="输入前全选已有文本。") + enter: bool = Field(default=False, description="输入后按回车。") + num_clicks: int = Field(default=1, description="鼠标点击次数。") + button_type: Literal["left", "middle", "right"] = Field(default="left", description="鼠标按键。") + hold_keys: list[str] = Field(default_factory=list, description="点击或拖拽时按住的功能键。") + keys: list[str] = Field(default_factory=list, description="快捷键按键 / 敲击按键。") + press_keys: list[str] = Field(default_factory=list, description="按住 hold_keys 时敲击的按键。") + clicks: int = Field(default=0, description="滚轮量，负数为向下。") + shift: bool = Field(default=False, description="水平滚动（True 时为横向）。") + wait_seconds: float = Field(default=0.5, description="等待秒数。") target_type: GUITargetType = Field( default="auto", - description=( - "Target UI element type for grounded actions. Use checkbox for agreement/privacy checkboxes, " - "input for editable text boxes, list_item for chat/contact/file rows, button for buttons, " - "link for clickable text links, or auto when unsure." - ), + description="目标 UI 元素类型：checkbox / input / list_item / button / link / auto。", ) class GUIToolInput(BaseModel): actions: list[GUIActionItem] = Field( default_factory=list, - description="Ordered list of GUI actions to execute sequentially. Each item specifies one GUI operation (click, type, drag_and_drop, scroll, hotkey, hold_and_press, wait, screenshot, move_to). If empty, returns current screenshot without performing any action.", + description="有序 GUI 操作列表，按顺序依次执行。空列表则仅返回当前截图。", ) +# ---------- 图片工具函数 ---------- def _get_gui_region() -> dict | None: """获取当前 GUI 操作显示器区域，失败返回 None（全屏）。""" try: @@ -104,10 +103,12 @@ def _get_gui_region() -> dict | None: return None -def _screenshot_png() -> tuple[bytes, tuple[int, int]]: +def _screenshot_png() -> Tuple[bytes, Tuple[int, int]]: region = _get_gui_region() if region: - image = pyautogui.screenshot(region=(region["x"], region["y"], region["width"], region["height"])) + image = pyautogui.screenshot( + region=(region["x"], region["y"], region["width"], region["height"]) + ) else: image = pyautogui.screenshot() width, height = image.size @@ -118,7 +119,10 @@ def _screenshot_png() -> tuple[bytes, tuple[int, int]]: def _resize_for_grounding(image_bytes: bytes) -> bytes: image = Image.open(io.BytesIO(image_bytes)).convert("RGB") - image = image.resize((env_utils.GROUNDING_WIDTH, env_utils.GROUNDING_HEIGHT), Image.Resampling.LANCZOS) + image = image.resize( + (env_utils.GROUNDING_WIDTH, env_utils.GROUNDING_HEIGHT), + Image.Resampling.LANCZOS, + ) buf = io.BytesIO() image.save(buf, format="PNG") return buf.getvalue() @@ -129,112 +133,50 @@ def _image_data_url(image_bytes: bytes) -> str: return f"data:image/png;base64,{encoded}" -def _parse_coordinates(text: str) -> tuple[int, int]: - numbers = re.findall(r"-?\d+(?:\.\d+)?", text or "") - if len(numbers) < 2: - raise ValueError(f"grounding model did not return coordinates: {text!r}") - x = int(float(numbers[0])) - y = int(float(numbers[1])) - x = max(0, min(env_utils.GROUNDING_WIDTH - 1, x)) - y = max(0, min(env_utils.GROUNDING_HEIGHT - 1, y)) - return x, y +# ---------- 坐标解析与转换 ---------- +def _parse_batch_coordinates(text: str) -> list[Tuple[int, int]]: + """解析批量坐标输出。 + + 按行提取坐标对，每行取第一对有效的 (x, y)。 + 兼容格式： + - "(50, 960)" + - "(100,200)" + - "1, (50, 960)" （带编号前缀） + - "1:(100,200)" （带编号前缀） + + 所有坐标会被裁剪到 [0, GROUNDING-1] × [0, GROUNDING-1]。 + """ + W, H = env_utils.GROUNDING_WIDTH - 1, env_utils.GROUNDING_HEIGHT - 1 + coordinates: list[Tuple[int, int]] = [] + + for line in text.strip().split("\n"): + line = line.strip() + if not line: + continue + matches = re.findall(r"\(\s*(-?\d+(?:\.\d+)?)\s*[,，]\s*(-?\d+(?:\.\d+)?)\s*\)", line) + if matches: + x = int(float(matches[0][0])) + y = int(float(matches[0][1])) + x = max(0, min(W, x)) + y = max(0, min(H, y)) + coordinates.append((x, y)) + + return coordinates -def _scale_to_screen(coords: tuple[int, int], screen_size: tuple[int, int]) -> tuple[int, int]: +def _scale_to_screen(coords: Tuple[int, int], screen_size: Tuple[int, int]) -> Tuple[int, int]: """将 grounding 坐标映射到绝对屏幕坐标（考虑 GUI 操作区域偏移）。""" screen_width, screen_height = screen_size x, y = coords local_x = round(x * screen_width / env_utils.GROUNDING_WIDTH) local_y = round(y * screen_height / env_utils.GROUNDING_HEIGHT) - # 加上目标显示器在屏幕空间的偏移 region = _get_gui_region() offset_x = region["x"] if region else 0 offset_y = region["y"] if region else 0 return (offset_x + local_x, offset_y + local_y) -def _target_type_instructions(target_type: str) -> str: - normalized = (target_type or "auto").strip().lower() - instructions = { - "checkbox": ( - "Target type: checkbox. Return the checkbox/radio circle or square itself, " - "and provide its center point. Do not click nearby agreement/privacy/service link text." - ), - "input": ( - "Target type: input. Return the editable text box area and provide a point " - "inside the editable area where typing will focus the field." - ), - "list_item": ( - "Target type: list_item. Return the full row/list item including icon/avatar " - "and label, and provide the row center point." - ), - "button": ( - "Target type: button. Return the full clickable button and provide its center point." - ), - "link": ( - "Target type: link. Return only the clickable text/link area and provide the text center." - ), - } - return instructions.get(normalized, "Target type: auto. Infer the UI element type from the query.") - - -def _infer_target_type(element_description: str, target_type: str) -> str: - normalized = (target_type or "auto").strip().lower() - if normalized != "auto": - return normalized - - desc = element_description or "" - checkbox_terms = ["复选框", "勾选框", "勾选", "已阅读", "同意", "协议", "隐私", "服务协议", "用户协议", "checkbox"] - if any(term in desc for term in checkbox_terms): - return "checkbox" - - input_terms = ["输入框", "文本框", "消息框", "编辑框", "输入栏", "input", "textbox"] - if any(term in desc for term in input_terms): - return "input" - - list_terms = ["列表", "联系人", "聊天列表", "文件列表", "这一行", "整行", "list"] - if any(term in desc for term in list_terms): - return "list_item" - - button_terms = ["按钮", "登录", "确定", "取消", "发送", "button"] - if any(term in desc for term in button_terms): - return "button" - - return "auto" - - -def _ground_element(element_description: str, target_type: str = "auto") -> tuple[int, int, str]: - if not element_description or not element_description.strip(): - raise ValueError("element_description is required for this GUI action.") - - target_type = _infer_target_type(element_description, target_type) - raw_png, screen_size = _screenshot_png() - grounding_png = _resize_for_grounding(raw_png) - prompt = ( - "You are a visual grounding model for desktop GUI control.\n" - f"The screenshot has been resized to exactly {env_utils.GROUNDING_WIDTH}x{env_utils.GROUNDING_HEIGHT} pixels.\n" - f"Query: {element_description.strip()}\n" - f"{_target_type_instructions(target_type)}\n" - "Return only one target point as pixel coordinates in this exact format: x,y\n" - f"Example: {env_utils.GROUNDING_WIDTH // 2},{env_utils.GROUNDING_HEIGHT // 2}\n" - "Do not include any explanation." - ) - response = _get_gui_llm().invoke( - [ - HumanMessage( - content=[ - {"type": "image_url", "image_url": {"url": _image_data_url(grounding_png)}}, - {"type": "text", "text": prompt}, - ] - ) - ] - ) - response_text = str(response.content) - gx, gy = _parse_coordinates(response_text) - x, y = _scale_to_screen((gx, gy), screen_size) - return x, y, f"grounded ({gx},{gy})" - - +# ---------- 文本输入辅助 ---------- def _type_text(text: str) -> None: if not text: return @@ -242,7 +184,6 @@ def _type_text(text: str) -> None: if has_unicode: try: import pyperclip - pyperclip.copy(text) modifier = "command" if sys.platform == "darwin" else "ctrl" pyautogui.hotkey(modifier, "v") @@ -268,7 +209,16 @@ def _with_hold_keys_up(keys: list[str]) -> None: pyautogui.keyUp(key) +# ===================================================================== +# 核心工具类 +# ===================================================================== class GUITool(BaseTool): + """GUI 自动化工具。 + + 坐标生成阶段采用批量方案：一次截图 + 全部 query 批量输入 LLM + 一次性解析所有坐标。 + N 个需要定位的动作 = 1 次 LLM 调用。 + """ + args_schema: type[BaseModel] = GUIToolInput name: str = "gui_tool" description: str = ( @@ -280,6 +230,18 @@ class GUITool(BaseTool): "会先视觉定位并点击目标输入框，然后再输入文本，无需单独调用 click。不提供 element_description 则直接在当前焦点输入。\n" "所有动作执行完毕后，返回当前屏幕的截图。\n" "\n" + "【网页滚动提示】浏览器中向下滚动网页推荐使用快捷键，而非 scroll 滚轮：" + "{\"action\":\"hotkey\",\"keys\":[\"space\"]}（空格键即 Page Down）。" + "向上滚动可使用 {\"action\":\"hotkey\",\"keys\":[\"shift\",\"space\"]}。\n" + "\n" + "【下拉框处理策略】操作下拉选择框时分两步：\n" + " 第1步（type 方式）：先尝试 {\"action\":\"type\",\"element_description\":\"目标下拉框\",\"text\":\"选项名称\"} 直接输入，\n" + " 能自动匹配则任务完成；\n" + " 第2步（键选方式）：若 type 不生效，则改为先点击展开下拉框，再用方向键选择，最后回车确认：\n" + " {\"action\":\"click\",\"element_description\":\"目标下拉框\"},\n" + " {\"action\":\"hotkey\",\"keys\":[\"down\"]}（按需多次 up/down 定位），\n" + " {\"action\":\"hotkey\",\"keys\":[\"enter\"]}。\n" + "\n" "示例：\n" "- 登录表单：{\"actions\": [" "{\"action\":\"type\",\"element_description\":\"用户名输入框\",\"text\":\"admin\"}," @@ -288,6 +250,11 @@ class GUITool(BaseTool): "- 浏览器搜索：{\"actions\": [" "{\"action\":\"type\",\"element_description\":\"浏览器地址栏\",\"text\":\"github.com\",\"enter\":true}" "]}\n" + "- 网页翻页浏览：{\"actions\": [" + "{\"action\":\"hotkey\",\"keys\":[\"space\"]}," + "{\"action\":\"wait\",\"wait_seconds\":1.0}," + "{\"action\":\"hotkey\",\"keys\":[\"space\"]}" + "]}\n" "- 等待并确认：{\"actions\": [" "{\"action\":\"click\",\"element_description\":\"提交按钮\"}," "{\"action\":\"wait\",\"wait_seconds\":2.0}" @@ -295,6 +262,143 @@ class GUITool(BaseTool): ) response_format: Literal["content", "content_and_artifact"] = "content_and_artifact" + # ---------- 批量坐标生成核心 ---------- + def _collect_grounding_queries(self, actions: list[GUIActionItem]) -> list[Tuple[int, str, str]]: + """遍历所有动作，收集需要视觉定位的 query。 + + 返回 [(action_index, query_type, query_text), ...] + query_type: "element" (元素描述) 或 "ending" (拖拽终点)。 + 对于 type 操作且无 element_description 的，不收集（无需定位）。 + """ + queries: list[Tuple[int, str, str]] = [] + for idx, act in enumerate(actions): + action = str(act.action).strip() + if action in ("screenshot", "wait", "hotkey", "hold_and_press"): + continue + + if action in ("click", "scroll", "move_to"): + desc = (act.element_description or "").strip() + if desc: + queries.append((idx, "element", desc)) + continue + + if action == "type": + desc = (act.element_description or "").strip() + if desc: + queries.append((idx, "element", desc)) + continue + + if action == "drag_and_drop": + start_desc = (act.element_description or "").strip() + end_desc = (act.ending_description or "").strip() + if start_desc: + queries.append((idx, "element", start_desc)) + if end_desc: + queries.append((idx, "ending", end_desc)) + continue + + return queries + + def _build_batch_prompt(self, queries: list[Tuple[int, str, str]]) -> str: + """构建包含 One-shot 示例的批量定位 prompt。""" + W = env_utils.GROUNDING_WIDTH + H = env_utils.GROUNDING_HEIGHT + + numbered = [] + for i, (act_idx, qtype, qtext) in enumerate(queries, 1): + label = f"action{act_idx + 1}_{qtype}" + numbered.append(f"{i}. [{label}] {qtext}") + query_text = "\n".join(numbered) + + prompt = ( + "You are a visual grounding model for desktop GUI control.\n" + f"The screenshot has been resized to exactly {W}x{H} pixels.\n" + "Your task is to locate MULTIPLE target elements in the same image in a single response.\n" + "\n" + "=== OUTPUT FORMAT (STRICT) ===\n" + "You MUST output one line per query in the EXACT format below:\n" + " (, )\n" + "Where:\n" + " - x: horizontal pixel coordinate (integer, 0 = leftmost)\n" + " - y: vertical pixel coordinate (integer, 0 = topmost)\n" + "\n" + "=== ONE-SHOT EXAMPLE ===\n" + "Scenario: A desktop screenshot ({W}x{H}) showing a browser with multiple UI elements.\n" + "\n" + "Queries:\n" + "1. [action1_element] the browser address bar at the top of the window\n" + "2. [action2_element] the search input field in the main content area\n" + "3. [action3_element] the search button next to the input field\n" + "4. [action3_ending] the first search result link after clicking search\n" + "\n" + "Expected output (four lines, one per query):\n" + f"({W // 2}, {H // 6})\n" + f"({W // 2}, {H // 3})\n" + f"({W // 2 + 200}, {H // 3})\n" + f"({W // 2}, {H // 2 + 50})\n" + "\n" + "=== NOW YOUR TURN ===\n" + "Locate the following elements in the provided screenshot:\n" + f"{query_text}\n" + "\n" + "=== RULES (FAILURE TO FOLLOW = INCORRECT) ===\n" + "1. Output exactly one line per query, in the SAME order (1, 2, 3, ...).\n" + "2. Each line format: `(, )` — nothing else on that line.\n" + "3. NO explanations, NO markdown, NO additional text before or after the coordinate lines.\n" + "4. Coordinates must be integers in range x=[0,{W-1}] y=[0,{H-1}].\n" + "5. If you genuinely cannot find an element, output your best reasonable guess for its coordinates.\n" + "6. Do NOT wrap the output in code fences (```), quotes, or JSON.\n" + "7. For a sequence of operations on the same page, generate coordinates strictly in action order. " + "Each coordinate pair corresponds to one step of the GUI operation sequence. " + "(对于一个页面中的操作需要按顺序进行单步多GUI操作)\n" + ) + return prompt + + def _batch_ground_elements( + self, queries: list[Tuple[int, str, str]], screen_size: Tuple[int, int] + ) -> dict[Tuple[int, str], Tuple[int, int]]: + """一次截图 + 一次 LLM 调用，批量生成所有目标元素的坐标。 + + 参数: + queries: [(action_index, query_type, query_text), ...] + screen_size: (screen_width, screen_height) + + 返回: + {(action_index, query_type): (screen_x, screen_y), ...} + """ + if not queries: + return {} + + # 1. 截图 + raw_png, _ = _screenshot_png() + grounding_png = _resize_for_grounding(raw_png) + + # 2. 构建 prompt + 单次 LLM 调用 + prompt = self._build_batch_prompt(queries) + response = _get_gui_llm().invoke( + [ + HumanMessage( + content=[ + {"type": "image_url", "image_url": {"url": _image_data_url(grounding_png)}}, + {"type": "text", "text": prompt}, + ] + ) + ] + ) + response_text = str(response.content).strip() + + # 3. 解析坐标 + coords = _parse_batch_coordinates(response_text) + + # 4. 建立映射 (action_index, query_type) → (screen_x, screen_y) + result: dict[Tuple[int, str], Tuple[int, int]] = {} + for i, (act_idx, qtype, _) in enumerate(queries): + if i < len(coords): + result[(act_idx, qtype)] = _scale_to_screen(coords[i], screen_size) + + return result + + # ---------- 截图 artifact ---------- def _artifact_screenshot(self) -> dict: try: image_bytes, _ = _screenshot_png() @@ -303,24 +407,35 @@ class GUITool(BaseTool): except Exception: return {} - def _run( - self, - actions: list[GUIActionItem] | None = None, - ) -> tuple[str, dict]: + # ---------- 顶层入口 ---------- + def _run(self, actions: list[GUIActionItem] | None = None) -> Tuple[str, dict]: try: - # ---------- 无动作时直接截图 ---------- if not actions: return ("", self._artifact_screenshot()) - # ---------- 顺序执行动作列表 ---------- - result_messages: list[str] = [] # 收集每个动作的返回信息 + # 1. 先截图一次，获取屏幕尺寸 + _, screen_size = _screenshot_png() + + # 2. 批量定位：一次 LLM 调用生成所有动作需要的坐标 + queries = self._collect_grounding_queries(actions) + coords_map = self._batch_ground_elements(queries, screen_size) + + # 3. 构建 (action_index, query_type) → (x, y) 查找表 + action_coords: dict[int, dict[str, Tuple[int, int]]] = {} + for (act_idx, qtype), coord in coords_map.items(): + action_coords.setdefault(act_idx, {})[qtype] = coord + + # 4. 顺序执行动作列表 + result_messages: list[str] = [] for idx, act in enumerate(actions, start=1): if act.action == "screenshot": result_messages.append(f"{idx}. Screenshot (skipped, final screenshot will be taken)") continue + act_idx = idx - 1 # 0-indexed + cached = action_coords.get(act_idx, {}) try: - msg = self._dispatch(**act.model_dump()) + msg = self._dispatch(cached_coords=cached, **act.model_dump()) result_messages.append(f"{idx}. {msg}") except Exception as exc: error_msg = f"{idx}. [GUI Error] {act.action}: {exc}" @@ -332,18 +447,19 @@ class GUITool(BaseTool): final_text = "\n".join(result_messages) return (final_text, self._artifact_screenshot()) except Exception as e: - return (f"[ERROR] gui_tool 工具执行出错: {str(e)}", {}) + return (f"[ERROR] gui_tool 执行出错: {str(e)}", {}) - def _dispatch(self, **kwargs: Any) -> str: + def _dispatch(self, cached_coords: dict[str, Tuple[int, int]], **kwargs: Any) -> str: + """分发到具体动作方法，传入已缓存的坐标（cached_coords = {"element": (x,y), "ending": (x,y)}）。""" action = str(kwargs["action"]).strip() if action == "click": - return self._click(**kwargs) + return self._click(cached_coords, **kwargs) if action == "type": - return self._type(**kwargs) + return self._type(cached_coords, **kwargs) if action == "drag_and_drop": - return self._drag_and_drop(**kwargs) + return self._drag_and_drop(cached_coords, **kwargs) if action == "scroll": - return self._scroll(**kwargs) + return self._scroll(cached_coords, **kwargs) if action == "hotkey": return self._hotkey(**kwargs) if action == "hold_and_press": @@ -353,34 +469,35 @@ class GUITool(BaseTool): time.sleep(seconds) return f"Waited {seconds:.2f} seconds." if action == "move_to": - return self._move_to(**kwargs) + return self._move_to(cached_coords, **kwargs) raise ValueError(f"unsupported GUI action: {action}") - def _click(self, **kwargs: Any) -> str: - x, y, grounding = _ground_element(kwargs["element_description"] or "", kwargs.get("target_type", "auto")) + # ---------- 各动作实现 ---------- + def _click(self, cached_coords: dict[str, Tuple[int, int]], **kwargs: Any) -> str: + desc = (kwargs.get("element_description") or "").strip() + if desc and "element" in cached_coords: + x, y = cached_coords["element"] + else: + x, y = pyautogui.position() hold_keys = kwargs["hold_keys"] _with_hold_keys_down(hold_keys) try: pyautogui.click(x, y, clicks=max(1, int(kwargs["num_clicks"])), button=kwargs["button_type"]) finally: _with_hold_keys_up(hold_keys) - return f"Clicked target. {grounding}" + return f"Clicked target at ({x},{y})." - def _type(self, **kwargs: Any) -> str: + def _type(self, cached_coords: dict[str, Tuple[int, int]], **kwargs: Any) -> str: element_desc = (kwargs.get("element_description") or "").strip() - if element_desc: - # 提供了目标元素描述：先定位点击再输入 - target_type = kwargs.get("target_type", "auto") - if target_type == "auto": - target_type = "input" # type 动作定位输入框时强制为 input - x, y, grounding = _ground_element(element_desc, target_type) + if element_desc and "element" in cached_coords: + x, y = cached_coords["element"] pyautogui.click(x, y) if kwargs["overwrite"]: _select_all_and_clear() _type_text(kwargs["text"]) if kwargs["enter"]: pyautogui.press("enter") - return f"Clicked target and typed text. {grounding}" + return f"Clicked target and typed text at ({x},{y})." # 不提供 element_description：直接在当前焦点输入 if kwargs["overwrite"]: _select_all_and_clear() @@ -389,9 +506,9 @@ class GUITool(BaseTool): pyautogui.press("enter") return "Typed text." - def _drag_and_drop(self, **kwargs: Any) -> str: - x1, y1, start_grounding = _ground_element(kwargs["element_description"] or "", kwargs.get("target_type", "auto")) - x2, y2, end_grounding = _ground_element(kwargs["ending_description"] or "", kwargs.get("target_type", "auto")) + def _drag_and_drop(self, cached_coords: dict[str, Tuple[int, int]], **kwargs: Any) -> str: + x1, y1 = cached_coords.get("element", pyautogui.position()) + x2, y2 = cached_coords.get("ending", pyautogui.position()) hold_keys = kwargs["hold_keys"] pyautogui.moveTo(x1, y1) _with_hold_keys_down(hold_keys) @@ -400,15 +517,15 @@ class GUITool(BaseTool): pyautogui.mouseUp() finally: _with_hold_keys_up(hold_keys) - return f"Dragged target. Start: {start_grounding}. End: {end_grounding}" + return f"Dragged from ({x1},{y1}) to ({x2},{y2})." - def _move_to(self, **kwargs: Any) -> str: - x, y, grounding = _ground_element(kwargs["element_description"] or "", kwargs.get("target_type", "auto")) + def _move_to(self, cached_coords: dict[str, Tuple[int, int]], **kwargs: Any) -> str: + x, y = cached_coords.get("element", pyautogui.position()) pyautogui.moveTo(x, y) - return f"Moved mouse to target. {grounding}" + return f"Moved mouse to ({x},{y})." - def _scroll(self, **kwargs: Any) -> str: - x, y, grounding = _ground_element(kwargs["element_description"] or "", kwargs.get("target_type", "auto")) + def _scroll(self, cached_coords: dict[str, Tuple[int, int]], **kwargs: Any) -> str: + x, y = cached_coords.get("element", pyautogui.position()) pyautogui.moveTo(x, y) time.sleep(0.1) clicks = int(kwargs["clicks"]) @@ -416,7 +533,7 @@ class GUITool(BaseTool): pyautogui.hscroll(clicks) else: pyautogui.vscroll(clicks) - return f"Scrolled target by {clicks}. {grounding}" + return f"Scrolled by {clicks} at ({x},{y})." def _hotkey(self, **kwargs: Any) -> str: keys = kwargs["keys"] @@ -437,5 +554,5 @@ class GUITool(BaseTool): _with_hold_keys_up(hold_keys) return f"Held {hold_keys} and pressed {press_keys}." - async def _arun(self, **kwargs: Any) -> tuple[str, dict]: - return await asyncio.to_thread(self._run, **kwargs) \ No newline at end of file + async def _arun(self, actions: list[GUIActionItem] | None = None) -> Tuple[str, dict]: + return await asyncio.to_thread(self._run, actions) diff --git a/src/agent/tools/tool.py b/src/agent/tools/tool.py index 6bbfcb9..2322f4f 100644 --- a/src/agent/tools/tool.py +++ b/src/agent/tools/tool.py @@ -211,7 +211,16 @@ class BrowserControl(BaseTool): # 无论操作是否成功，都尝试截屏（等待窗口切换） time.sleep(0.3) try: - pil_img = pyautogui.screenshot() + # 先尝试获取 GUI 操作区域，限定截屏范围 + try: + from agent.core.config_manager import get_active_monitor_region + region = get_active_monitor_region() + except Exception: + region = None + if region: + pil_img = pyautogui.screenshot(region=(region["x"], region["y"], region["width"], region["height"])) + else: + pil_img = pyautogui.screenshot() pil_img = pil_img.resize((env_utils.GROUNDING_WIDTH, env_utils.GROUNDING_HEIGHT), Image.Resampling.LANCZOS) buf = io.BytesIO() pil_img.save(buf, format="PNG")