849 lines
39 KiB
Python
849 lines
39 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
from .memory import Blackboard
|
|
from .model_client import ModelClient
|
|
from .prompts import APP_AGENT_SYSTEM_PROMPT, HOST_AGENT_SYSTEM_PROMPT, SYSTEM_TEST_ANALYST_PROMPT
|
|
from .tools import AgentToolRegistry, ToolResult, normalize_url
|
|
|
|
|
|
SubmitTask = Callable[[dict[str, Any]], dict[str, Any]]
|
|
|
|
|
|
@dataclass
|
|
class AgentSessionResult:
|
|
payload: dict[str, Any]
|
|
|
|
|
|
class AgentSession:
|
|
"""
|
|
Minimal UFO-inspired execution loop for one claimed task.
|
|
|
|
The implementation keeps the same conceptual boundaries as UFO:
|
|
HostAgent decides the subtask, AppAgent performs one safe step, Blackboard
|
|
persists requests/observations/trajectories for later rounds.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
task: dict[str, Any],
|
|
model_config_state: dict[str, Any],
|
|
memory_path: Path,
|
|
request_timeout_seconds: float,
|
|
submit_task: SubmitTask | None = None,
|
|
) -> None:
|
|
self.task = task
|
|
self.model_config_state = model_config_state
|
|
self.model_config = model_config_state.get("value") if isinstance(model_config_state.get("value"), dict) else {}
|
|
self.request_timeout_seconds = request_timeout_seconds
|
|
self.submit_task = submit_task
|
|
self.blackboard = Blackboard(memory_path)
|
|
self.tools = AgentToolRegistry()
|
|
self.model = ModelClient(self.model_config, timeout_seconds=self._model_timeout_seconds())
|
|
self.logs: list[str] = []
|
|
self.screenshot_dir = memory_path.parent / "screenshots"
|
|
|
|
def _model_timeout_seconds(self) -> float:
|
|
try:
|
|
timeout = float(self.model_config.get("request_timeout_seconds") or self.request_timeout_seconds)
|
|
except (TypeError, ValueError):
|
|
timeout = self.request_timeout_seconds
|
|
return max(1.0, min(timeout, 300.0))
|
|
|
|
@property
|
|
def payload(self) -> dict[str, Any]:
|
|
value = self.task.get("payload")
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
@property
|
|
def task_id(self) -> str:
|
|
return str(self.task.get("external_id") or self.task.get("id") or "")
|
|
|
|
def target_url(self) -> str:
|
|
return normalize_url(
|
|
self.payload.get("target_url")
|
|
or self.payload.get("url")
|
|
or self.payload.get("site_url")
|
|
or self.task.get("target_url")
|
|
or self.model_config.get("target_system_url")
|
|
)
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
started = time.monotonic()
|
|
self.blackboard.append(
|
|
"requests",
|
|
{
|
|
"task": self._task_brief(),
|
|
"model_config_version": self.model_config_state.get("version"),
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
|
|
host_decision = self._host_decision()
|
|
if self.execution_mode() == "gui":
|
|
return self._run_gui_task(started, host_decision)
|
|
|
|
target_url = self.target_url()
|
|
if not target_url:
|
|
return self._finish_without_target(started, host_decision)
|
|
|
|
open_browser_result: ToolResult | None = None
|
|
if self.payload.get("open_in_browser", True):
|
|
open_browser_result = self.tools.web.open_in_browser(
|
|
target_url,
|
|
wait_seconds=float(self.payload.get("browser_wait_seconds") or 1.5),
|
|
)
|
|
self.blackboard.append(
|
|
"actions",
|
|
{
|
|
"tool": "open_in_browser",
|
|
"ok": open_browser_result.ok,
|
|
"error": open_browser_result.error,
|
|
"result": open_browser_result.result,
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
if open_browser_result.ok:
|
|
self.logs.append(f"Opened target URL in browser: {target_url}")
|
|
else:
|
|
self.logs.append(f"Browser open failed: {open_browser_result.error}")
|
|
|
|
observation_result = self.tools.web.observe_page(
|
|
target_url,
|
|
timeout_seconds=self.request_timeout_seconds,
|
|
)
|
|
observation = observation_result.result
|
|
desktop_screenshot_result: ToolResult | None = None
|
|
if self.payload.get("capture_screenshot", True):
|
|
desktop_screenshot_result = self.tools.gui.capture_desktop(
|
|
self.screenshot_dir,
|
|
prefix=f"task_{self.task.get('id') or self.task_id}_web",
|
|
)
|
|
if desktop_screenshot_result.ok:
|
|
self.blackboard.append(
|
|
"screenshots",
|
|
{
|
|
"image_path": desktop_screenshot_result.result.get("image_path"),
|
|
"scope": "web_desktop",
|
|
"target_url": target_url,
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
elif desktop_screenshot_result.error:
|
|
self.logs.append(f"Desktop screenshot failed: {desktop_screenshot_result.error}")
|
|
|
|
self.blackboard.append(
|
|
"observations",
|
|
{
|
|
"tool": "observe_web_page",
|
|
"ok": observation_result.ok,
|
|
"error": observation_result.error,
|
|
"browser_open": open_browser_result.result if open_browser_result else None,
|
|
"screenshot": desktop_screenshot_result.result if desktop_screenshot_result and desktop_screenshot_result.ok else None,
|
|
"summary": self._observation_summary(observation),
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
|
|
app_decision = self._app_decision(host_decision, observation_result)
|
|
discovered_children: list[dict[str, Any]] = []
|
|
model_generated_children: list[dict[str, Any]] = []
|
|
rule_discovered_children: list[dict[str, Any]] = []
|
|
submitted_children: list[dict[str, Any]] = []
|
|
model_analysis: dict[str, Any] = {
|
|
"model_used": False,
|
|
"model_available": self.model.available,
|
|
}
|
|
discovery_source = "none"
|
|
if self._allow_child_discovery(observation_result):
|
|
model_analysis, model_generated_children = self._model_discover_child_tasks(observation_result)
|
|
if model_generated_children:
|
|
discovered_children = model_generated_children
|
|
discovery_source = "llm"
|
|
else:
|
|
rule_discovered_children = self.tools.web.discover_child_tasks(
|
|
self.task,
|
|
observation,
|
|
max_depth=int(self.payload.get("discovery_max_depth") or 2),
|
|
)
|
|
discovered_children = rule_discovered_children
|
|
discovery_source = "rule"
|
|
if model_analysis.get("model_available") and model_analysis.get("model_used"):
|
|
self.logs.append("Model returned no usable functional tasks; used rule-based fallback.")
|
|
submitted_children = self._submit_child_tasks(discovered_children)
|
|
|
|
success, message, needs_user_input = self._status_from_observation(observation_result, app_decision)
|
|
if model_analysis.get("needs_user_input"):
|
|
needs_user_input = True
|
|
success = False
|
|
message = str(model_analysis.get("user_input_reason") or "Model determined that user input is required.")
|
|
duration = round(time.monotonic() - started, 3)
|
|
result = {
|
|
"success": success,
|
|
"message": message,
|
|
"task_type": self.payload.get("task_type") or "manual",
|
|
"agent_framework": "ufo-inspired-lite",
|
|
"p0_mode": True,
|
|
"needs_user_input": needs_user_input,
|
|
"target_url": target_url,
|
|
"final_url": observation.get("final_url"),
|
|
"http_status": observation.get("http_status"),
|
|
"content_type": observation.get("content_type"),
|
|
"page_title": observation.get("page_title"),
|
|
"duration_seconds": duration,
|
|
"action_duration_seconds": round(
|
|
observation_result.duration_seconds
|
|
+ (open_browser_result.duration_seconds if open_browser_result else 0)
|
|
+ (desktop_screenshot_result.duration_seconds if desktop_screenshot_result else 0),
|
|
3,
|
|
),
|
|
"model_duration_seconds": round(
|
|
self._model_duration(host_decision, app_decision)
|
|
+ float(model_analysis.get("model_duration_seconds") or 0),
|
|
3,
|
|
),
|
|
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
|
"host_decision": host_decision,
|
|
"app_decision": app_decision,
|
|
"model_available": self.model.available,
|
|
"model_analysis": model_analysis,
|
|
"model_errors": self._model_errors(host_decision, app_decision, model_analysis),
|
|
"observation": self._observation_summary(observation),
|
|
"browser_open": open_browser_result.result if open_browser_result else None,
|
|
"screenshots": [
|
|
desktop_screenshot_result.result
|
|
] if desktop_screenshot_result and desktop_screenshot_result.ok else [],
|
|
"available_tools": self.tools.descriptions(),
|
|
"discovered_child_tasks": self._child_task_briefs(discovered_children),
|
|
"model_generated_child_tasks": self._child_task_briefs(model_generated_children),
|
|
"rule_discovered_child_tasks": self._child_task_briefs(rule_discovered_children),
|
|
"discovery_source": discovery_source,
|
|
"submitted_child_tasks": submitted_children,
|
|
"memory": self.blackboard.summary(),
|
|
"logs": [*self.logs, *(observation_result.result.get("logs") or [])],
|
|
}
|
|
self.blackboard.append(
|
|
"trajectories",
|
|
{
|
|
"host_decision": host_decision,
|
|
"app_decision": app_decision,
|
|
"result": {
|
|
"success": success,
|
|
"message": message,
|
|
"submitted_child_count": len(submitted_children),
|
|
"duration_seconds": duration,
|
|
},
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
result["memory"] = self.blackboard.summary()
|
|
return result
|
|
|
|
def execution_mode(self) -> str:
|
|
explicit = str(self.payload.get("execution_mode") or self.payload.get("agent_mode") or "").strip().lower()
|
|
if explicit in {"web", "gui"}:
|
|
return explicit
|
|
task_type = str(self.payload.get("task_type") or "").lower()
|
|
if task_type.startswith("gui") or self.payload.get("window_id") or self.payload.get("window_title"):
|
|
return "gui"
|
|
return "web"
|
|
|
|
def _run_gui_task(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]:
|
|
windows_result = self.tools.gui.list_windows()
|
|
screenshot_result = self.tools.gui.capture_desktop(self.screenshot_dir, prefix=f"task_{self.task.get('id') or self.task_id}_desktop")
|
|
selected_window = self._select_window(windows_result.result.get("windows") or [])
|
|
inspect_result: ToolResult | None = None
|
|
window_screenshot_result: ToolResult | None = None
|
|
if selected_window:
|
|
inspect_result = self.tools.gui.inspect_window(str(selected_window["id"]))
|
|
window_screenshot_result = self.tools.gui.capture_window(
|
|
str(selected_window["id"]),
|
|
self.screenshot_dir,
|
|
prefix=f"task_{self.task.get('id') or self.task_id}_window",
|
|
)
|
|
|
|
observation = self._gui_observation_summary(
|
|
windows_result,
|
|
screenshot_result,
|
|
selected_window,
|
|
inspect_result,
|
|
window_screenshot_result,
|
|
)
|
|
self.blackboard.append(
|
|
"observations",
|
|
{
|
|
"tool": "gui_observe",
|
|
"ok": windows_result.ok,
|
|
"error": windows_result.error,
|
|
"summary": observation,
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
if screenshot_result.ok:
|
|
self.blackboard.append(
|
|
"screenshots",
|
|
{"image_path": screenshot_result.result.get("image_path"), "scope": "desktop"},
|
|
task_id=self.task_id,
|
|
)
|
|
if window_screenshot_result and window_screenshot_result.ok:
|
|
self.blackboard.append(
|
|
"screenshots",
|
|
{"image_path": window_screenshot_result.result.get("image_path"), "scope": "window", "window": selected_window},
|
|
task_id=self.task_id,
|
|
)
|
|
|
|
app_decision = {
|
|
"observation": json.dumps(observation, ensure_ascii=False),
|
|
"thought": "GUI P0 阶段只枚举窗口、控件并保存证据截图，不执行点击或输入。",
|
|
"action": {
|
|
"function": "list_windows" if not selected_window else "inspect_window_controls",
|
|
"arguments": {
|
|
"window_id": selected_window.get("id") if selected_window else None,
|
|
"window_title": self.payload.get("window_title"),
|
|
},
|
|
"status": "FINISH" if windows_result.ok and (not self.payload.get("window_title") or selected_window) else "FAIL",
|
|
},
|
|
"plan": [],
|
|
"comment": "GUI 观察完成。" if windows_result.ok else windows_result.error,
|
|
"save_screenshot": {
|
|
"save": bool(screenshot_result.ok or (window_screenshot_result and window_screenshot_result.ok)),
|
|
"reason": "GUI P0 观察需要截图作为证据。" if screenshot_result.ok else screenshot_result.error,
|
|
},
|
|
"result": "GUI 窗口和控件信息已记录。" if windows_result.ok else windows_result.error,
|
|
"model_used": False,
|
|
}
|
|
|
|
success = windows_result.ok and (not self.payload.get("window_title") or bool(selected_window))
|
|
message = "Agent GUI P0 observation completed." if success else "Requested GUI window was not found or could not be inspected."
|
|
duration = round(time.monotonic() - started, 3)
|
|
result = {
|
|
"success": success,
|
|
"message": message,
|
|
"task_type": self.payload.get("task_type") or "gui_p0",
|
|
"agent_framework": "ufo-inspired-lite",
|
|
"p0_mode": True,
|
|
"execution_mode": "gui",
|
|
"needs_user_input": False,
|
|
"duration_seconds": duration,
|
|
"action_duration_seconds": round(
|
|
windows_result.duration_seconds
|
|
+ screenshot_result.duration_seconds
|
|
+ (inspect_result.duration_seconds if inspect_result else 0)
|
|
+ (window_screenshot_result.duration_seconds if window_screenshot_result else 0),
|
|
3,
|
|
),
|
|
"model_duration_seconds": self._model_duration(host_decision, app_decision),
|
|
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
|
"host_decision": host_decision,
|
|
"app_decision": app_decision,
|
|
"observation": observation,
|
|
"available_tools": self.tools.descriptions(),
|
|
"memory": self.blackboard.summary(),
|
|
"logs": [
|
|
*self.logs,
|
|
*(windows_result.error and [windows_result.error] or []),
|
|
*(screenshot_result.error and [screenshot_result.error] or []),
|
|
*(inspect_result and inspect_result.error and [inspect_result.error] or []),
|
|
],
|
|
}
|
|
self.blackboard.append(
|
|
"trajectories",
|
|
{
|
|
"host_decision": host_decision,
|
|
"app_decision": app_decision,
|
|
"result": {
|
|
"success": success,
|
|
"message": message,
|
|
"duration_seconds": duration,
|
|
},
|
|
},
|
|
task_id=self.task_id,
|
|
)
|
|
result["memory"] = self.blackboard.summary()
|
|
return result
|
|
|
|
def _host_decision(self) -> dict[str, Any]:
|
|
fallback = {
|
|
"observation": "收到一个主控端分配的测试任务。",
|
|
"thought": "P0 阶段先把任务限定到单个 Web/AppAgent，执行无副作用观察，并按页面结构生成后续子任务。",
|
|
"current_subtask": self.payload.get("objective") or self.payload.get("instruction") or self.task.get("title") or "",
|
|
"message": ["只执行无副作用观察动作；遇到登录、验证码、权限或破坏性入口时暂停并回传。"],
|
|
"status": "ASSIGN",
|
|
"plan": ["观察目标页面", "记录页面元信息", "识别可继续拆分的链接、表单和入口"],
|
|
"function": "assign_app_agent",
|
|
"arguments": {"agent": "web_app_agent", "target_url": self.target_url()},
|
|
"questions": [],
|
|
"result": "已分配给 Web AppAgent 执行 P0 观察。",
|
|
"model_used": False,
|
|
}
|
|
if not self.model.available:
|
|
return fallback
|
|
|
|
context = {
|
|
"task": self._task_brief(),
|
|
"blackboard": self.blackboard.snapshot(limit=5),
|
|
"tools": self.tools.descriptions(),
|
|
}
|
|
response = self.model.chat_json(
|
|
[
|
|
{"role": "system", "content": HOST_AGENT_SYSTEM_PROMPT},
|
|
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
|
|
]
|
|
)
|
|
if response.ok and isinstance(response.parsed, dict):
|
|
parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds}
|
|
return parsed
|
|
self.logs.append(f"HostAgent model fallback: {response.error or 'invalid JSON response'}")
|
|
return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds}
|
|
|
|
def _app_decision(self, host_decision: dict[str, Any], observation_result: ToolResult) -> dict[str, Any]:
|
|
observation = self._observation_summary(observation_result.result)
|
|
fallback_status = "FINISH" if observation_result.ok else "FAIL"
|
|
if observation.get("requires_auth"):
|
|
fallback_status = "PENDING"
|
|
fallback = {
|
|
"observation": json.dumps(observation, ensure_ascii=False),
|
|
"thought": "P0 阶段采用 observe_web_page 工具完成可达性和页面结构观察，不执行提交、保存、删除等有副作用动作。",
|
|
"action": {
|
|
"function": "observe_web_page",
|
|
"arguments": {"target_url": self.target_url()},
|
|
"status": fallback_status,
|
|
},
|
|
"plan": [],
|
|
"comment": "页面观察完成。" if observation_result.ok else observation_result.error,
|
|
"save_screenshot": {"save": False, "reason": "P0 Web 观察暂不保存截图。"},
|
|
"result": "页面可达性和结构信息已记录。" if observation_result.ok else observation_result.error,
|
|
"model_used": False,
|
|
}
|
|
if not self.model.available:
|
|
return fallback
|
|
|
|
context = {
|
|
"host_decision": host_decision,
|
|
"task": self._task_brief(),
|
|
"observation": observation,
|
|
"available_tools": self.tools.descriptions(),
|
|
"recent_trajectories": self.blackboard.recent("trajectories", 5),
|
|
}
|
|
response = self.model.chat_json(
|
|
[
|
|
{"role": "system", "content": APP_AGENT_SYSTEM_PROMPT},
|
|
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
|
|
]
|
|
)
|
|
if response.ok and isinstance(response.parsed, dict):
|
|
parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds}
|
|
return parsed
|
|
self.logs.append(f"AppAgent model fallback: {response.error or 'invalid JSON response'}")
|
|
return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds}
|
|
|
|
def _model_discover_child_tasks(self, observation_result: ToolResult) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
base_analysis: dict[str, Any] = {
|
|
"model_available": self.model.available,
|
|
"model_called": False,
|
|
"model_call_succeeded": False,
|
|
"model_used": False,
|
|
"model_duration_seconds": 0,
|
|
"summary": "",
|
|
"coverage_notes": [],
|
|
"needs_user_input": False,
|
|
"user_input_reason": "",
|
|
"functional_task_count": 0,
|
|
}
|
|
if not observation_result.ok:
|
|
return {**base_analysis, "error": observation_result.error or "Page observation failed."}, []
|
|
if not self.model.available:
|
|
self.logs.append("Model config is incomplete; used rule-based discovery fallback.")
|
|
return {**base_analysis, "error": "Model config is incomplete."}, []
|
|
|
|
observation = observation_result.result
|
|
context = {
|
|
"task": self._task_brief(),
|
|
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
|
"page": self._observation_for_model(observation),
|
|
"recent_memory": self.blackboard.snapshot(limit=4),
|
|
"constraints": {
|
|
"p0_no_side_effects": True,
|
|
"max_functional_tasks": 8,
|
|
"discovery_max_depth": int(self.payload.get("discovery_max_depth") or 2),
|
|
},
|
|
}
|
|
response = self.model.chat_json(
|
|
[
|
|
{"role": "system", "content": SYSTEM_TEST_ANALYST_PROMPT},
|
|
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
|
|
]
|
|
)
|
|
if not response.ok or not isinstance(response.parsed, dict):
|
|
self.logs.append(f"SystemTestAnalyst model fallback: {response.error or 'invalid JSON response'}")
|
|
return {
|
|
**base_analysis,
|
|
"model_called": True,
|
|
"model_call_succeeded": False,
|
|
"model_used": False,
|
|
"model_duration_seconds": response.duration_seconds,
|
|
"error": response.error or "Model did not return a valid JSON object.",
|
|
}, []
|
|
|
|
parsed = response.parsed
|
|
children = self._children_from_model_analysis(parsed, observation)
|
|
analysis = {
|
|
**base_analysis,
|
|
"model_called": True,
|
|
"model_call_succeeded": True,
|
|
"model_used": True,
|
|
"model_duration_seconds": response.duration_seconds,
|
|
"summary": str(parsed.get("summary") or ""),
|
|
"coverage_notes": parsed.get("coverage_notes") if isinstance(parsed.get("coverage_notes"), list) else [],
|
|
"needs_user_input": bool(parsed.get("needs_user_input")),
|
|
"user_input_reason": str(parsed.get("user_input_reason") or ""),
|
|
"functional_task_count": len(children),
|
|
}
|
|
self.logs.append(
|
|
"Model analyzed page and generated "
|
|
f"{len(children)} functional task(s): {analysis.get('summary') or 'no summary'}"
|
|
)
|
|
return analysis, children
|
|
|
|
def _observation_for_model(self, observation: dict[str, Any]) -> dict[str, Any]:
|
|
def clean_items(items: Any, limit: int) -> list[dict[str, Any]]:
|
|
if not isinstance(items, list):
|
|
return []
|
|
cleaned: list[dict[str, Any]] = []
|
|
for item in items[:limit]:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
cleaned.append(
|
|
{
|
|
key: value
|
|
for key, value in item.items()
|
|
if key in {"href", "text", "action", "method", "id", "name", "class", "type", "fields", "placeholder", "aria_label", "in_form"}
|
|
}
|
|
)
|
|
return cleaned
|
|
|
|
return {
|
|
"url": observation.get("url"),
|
|
"final_url": observation.get("final_url"),
|
|
"http_status": observation.get("http_status"),
|
|
"content_type": observation.get("content_type"),
|
|
"page_title": observation.get("page_title"),
|
|
"requires_auth": observation.get("requires_auth"),
|
|
"has_login_form": observation.get("has_login_form"),
|
|
"links": clean_items(observation.get("links"), 20),
|
|
"forms": clean_items(observation.get("forms"), 12),
|
|
"buttons": clean_items(observation.get("buttons"), 20),
|
|
}
|
|
|
|
def _children_from_model_analysis(self, analysis: dict[str, Any], observation: dict[str, Any]) -> list[dict[str, Any]]:
|
|
raw_items = analysis.get("functional_tasks")
|
|
if not isinstance(raw_items, list):
|
|
return []
|
|
max_children = 8
|
|
children: list[dict[str, Any]] = []
|
|
seen: set[tuple[str, str]] = set()
|
|
for index, item in enumerate(raw_items[:max_children], start=1):
|
|
if not isinstance(item, dict):
|
|
continue
|
|
child = self._child_from_model_item(item, observation, index)
|
|
key = (
|
|
str(child.get("title") or "").strip().lower(),
|
|
str((child.get("payload") or {}).get("target_url") or "").strip().lower(),
|
|
)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
children.append(child)
|
|
return children
|
|
|
|
def _child_from_model_item(self, item: dict[str, Any], observation: dict[str, Any], index: int) -> dict[str, Any]:
|
|
parent_external_id = str(self.task.get("external_id") or self.task.get("id") or "")
|
|
root_external_id = str(self.task.get("root_external_id") or self.payload.get("root_external_id") or parent_external_id)
|
|
parent_depth = int(self.task.get("depth") or self.payload.get("depth") or 0)
|
|
depth = parent_depth + 1
|
|
discovery_max_depth = int(self.payload.get("discovery_max_depth") or 2)
|
|
base_url = observation.get("final_url") or observation.get("url") or self.target_url()
|
|
raw_target_url = str(item.get("target_url") or "").strip()
|
|
target_url = urljoin(base_url, raw_target_url) if raw_target_url else base_url
|
|
title = str(item.get("title") or f"模型生成测试任务 {index}").strip()
|
|
objective = str(item.get("objective") or title).strip()
|
|
module = str(item.get("module") or title).strip()
|
|
level = str(item.get("level") or "feature").strip().lower()
|
|
if level not in {"module", "feature", "case"}:
|
|
level = "feature"
|
|
risk = str(item.get("risk_level") or "low").strip().lower()
|
|
if risk not in {"low", "medium", "high"}:
|
|
risk = "low"
|
|
try:
|
|
priority = int(item.get("priority") or 30 + depth * 10 + index)
|
|
except (TypeError, ValueError):
|
|
priority = 30 + depth * 10 + index
|
|
preconditions = item.get("preconditions") if isinstance(item.get("preconditions"), list) else []
|
|
external_id = self._stable_child_id(root_external_id, parent_external_id, title, target_url, objective)
|
|
payload = {
|
|
"task_type": "smart_discovery_p0",
|
|
"instruction": objective,
|
|
"objective": objective,
|
|
"target_url": target_url,
|
|
"source": "llm_analysis",
|
|
"llm_reason": item.get("reason") or "",
|
|
"level": level,
|
|
"depth": depth,
|
|
"risk_level": risk,
|
|
"allow_child_discovery": level in {"module", "feature"} and depth < discovery_max_depth,
|
|
"discovery_max_depth": discovery_max_depth,
|
|
"open_in_browser": True,
|
|
"capture_screenshot": True,
|
|
"preconditions": preconditions,
|
|
"hierarchy": {
|
|
"id": external_id,
|
|
"parent_id": parent_external_id,
|
|
"root_id": root_external_id,
|
|
"depth": depth,
|
|
"level": level,
|
|
},
|
|
}
|
|
return {
|
|
"external_id": external_id,
|
|
"title": title,
|
|
"module": module,
|
|
"parent_external_id": parent_external_id,
|
|
"root_external_id": root_external_id,
|
|
"depth": depth,
|
|
"task_level": level,
|
|
"risk_level": risk,
|
|
"priority": priority,
|
|
"failure_policy": "pause" if risk in {"medium", "high"} or preconditions else "skip",
|
|
"max_retries": 2,
|
|
"timeout_seconds": 90,
|
|
"tags": ["llm-generated", "system-test", "p0"],
|
|
"payload": payload,
|
|
}
|
|
|
|
def _stable_child_id(self, *parts: Any) -> str:
|
|
raw = "|".join(str(part or "") for part in parts)
|
|
return f"llm-{hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]}"
|
|
|
|
def _allow_child_discovery(self, observation_result: ToolResult) -> bool:
|
|
if not observation_result.ok:
|
|
return False
|
|
allow = self.payload.get("allow_child_discovery")
|
|
if allow is not None:
|
|
return bool(allow)
|
|
if self.payload.get("task_type") != "smart_discovery_p0":
|
|
return False
|
|
level = str(self.payload.get("level") or self.task.get("task_level") or "")
|
|
depth = int(self.payload.get("depth") or self.task.get("depth") or 0)
|
|
return level in {"site", "module", ""} and depth < int(self.payload.get("discovery_max_depth") or 2)
|
|
|
|
def _submit_child_tasks(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
if not children or self.submit_task is None:
|
|
return []
|
|
submitted: list[dict[str, Any]] = []
|
|
plan_name = self.task.get("plan_name") or self.payload.get("plan_name") or "智能探索 P0"
|
|
for child in children:
|
|
payload = {**child, "plan_name": plan_name}
|
|
try:
|
|
created = self.submit_task(payload)
|
|
except Exception as exc:
|
|
self.logs.append(f"Submit child task failed: {child.get('external_id')} error={exc}")
|
|
continue
|
|
submitted.append(
|
|
{
|
|
"external_id": created.get("external_id") or child.get("external_id"),
|
|
"title": created.get("title") or child.get("title"),
|
|
"depth": created.get("depth") if created.get("depth") is not None else child.get("depth"),
|
|
"task_level": created.get("task_level") or child.get("task_level"),
|
|
}
|
|
)
|
|
return submitted
|
|
|
|
def _status_from_observation(self, observation_result: ToolResult, app_decision: dict[str, Any]) -> tuple[bool, str, bool]:
|
|
action = app_decision.get("action") if isinstance(app_decision.get("action"), dict) else {}
|
|
status = str(action.get("status") or "").upper()
|
|
if status == "PENDING":
|
|
return False, "Task needs user input before it can continue.", True
|
|
if status in {"FAIL", "CONFIRM"}:
|
|
return False, app_decision.get("result") or app_decision.get("comment") or "Agent action did not complete.", status == "CONFIRM"
|
|
if not observation_result.ok:
|
|
return False, observation_result.error or "Target observation failed.", False
|
|
if observation_result.result.get("http_status") in {401, 403}:
|
|
return False, "Target page requires authentication or permission.", True
|
|
return True, "Agent P0 observation completed.", False
|
|
|
|
def _finish_without_target(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]:
|
|
task_type = self.payload.get("task_type") or "manual"
|
|
success = task_type != "smart_discovery_p0"
|
|
duration = round(time.monotonic() - started, 3)
|
|
result = {
|
|
"success": success,
|
|
"message": "No executable target URL was provided." if not success else "Task accepted; no executable URL was provided for P0 mode.",
|
|
"task_type": task_type,
|
|
"agent_framework": "ufo-inspired-lite",
|
|
"p0_mode": True,
|
|
"needs_user_input": not success,
|
|
"duration_seconds": duration,
|
|
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
|
"host_decision": host_decision,
|
|
"app_decision": {
|
|
"action": {"function": "no_action", "arguments": {}, "status": "FAIL" if not success else "FINISH"},
|
|
"result": "Missing target URL.",
|
|
},
|
|
"available_tools": self.tools.descriptions(),
|
|
"memory": self.blackboard.summary(),
|
|
"logs": ["AgentSession requires target_url for Web P0 execution."],
|
|
}
|
|
self.blackboard.append("trajectories", {"host_decision": host_decision, "result": result}, task_id=self.task_id)
|
|
return result
|
|
|
|
def _select_window(self, windows: list[dict[str, Any]]) -> dict[str, Any] | None:
|
|
window_id = self.payload.get("window_id")
|
|
if window_id:
|
|
for window in windows:
|
|
if str(window.get("id")) == str(window_id) or str(window.get("hex_id")).lower() == str(window_id).lower():
|
|
return window
|
|
return None
|
|
|
|
title = str(self.payload.get("window_title") or self.payload.get("application") or "").strip().lower()
|
|
if not title:
|
|
return None
|
|
for window in windows:
|
|
if title in str(window.get("title") or "").lower():
|
|
return window
|
|
for window in windows:
|
|
if title in str(window.get("class_name") or "").lower():
|
|
return window
|
|
return None
|
|
|
|
def _gui_observation_summary(
|
|
self,
|
|
windows_result: ToolResult,
|
|
screenshot_result: ToolResult,
|
|
selected_window: dict[str, Any] | None,
|
|
inspect_result: ToolResult | None,
|
|
window_screenshot_result: ToolResult | None,
|
|
) -> dict[str, Any]:
|
|
windows = windows_result.result.get("windows") if isinstance(windows_result.result, dict) else []
|
|
controls = []
|
|
if inspect_result and isinstance(inspect_result.result, dict):
|
|
controls = inspect_result.result.get("controls") or []
|
|
return {
|
|
"backend": (windows_result.result or {}).get("backend") or self.tools.gui.backend_status(),
|
|
"window_count": len(windows or []),
|
|
"windows": [
|
|
{
|
|
"id": window.get("id"),
|
|
"title": window.get("title"),
|
|
"class_name": window.get("class_name"),
|
|
"process_id": window.get("process_id"),
|
|
"rect": window.get("rect"),
|
|
"minimized": window.get("minimized"),
|
|
}
|
|
for window in (windows or [])[:20]
|
|
],
|
|
"selected_window": selected_window,
|
|
"control_count": len(controls),
|
|
"controls": [
|
|
{
|
|
"id": control.get("id"),
|
|
"text": control.get("text"),
|
|
"class_name": control.get("class_name"),
|
|
"rect": control.get("rect"),
|
|
"visible": control.get("visible"),
|
|
"enabled": control.get("enabled"),
|
|
}
|
|
for control in controls[:80]
|
|
],
|
|
"desktop_screenshot": screenshot_result.result if screenshot_result.ok else None,
|
|
"window_screenshot": window_screenshot_result.result if window_screenshot_result and window_screenshot_result.ok else None,
|
|
"errors": [
|
|
message
|
|
for message in [
|
|
windows_result.error,
|
|
screenshot_result.error,
|
|
inspect_result.error if inspect_result else "",
|
|
window_screenshot_result.error if window_screenshot_result else "",
|
|
]
|
|
if message
|
|
],
|
|
}
|
|
|
|
def _task_brief(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.task.get("id"),
|
|
"external_id": self.task.get("external_id"),
|
|
"title": self.task.get("title"),
|
|
"plan_name": self.task.get("plan_name"),
|
|
"module": self.task.get("module"),
|
|
"status": self.task.get("status"),
|
|
"depth": self.task.get("depth"),
|
|
"task_level": self.task.get("task_level"),
|
|
"risk_level": self.task.get("risk_level"),
|
|
"payload": self.payload,
|
|
}
|
|
|
|
def _task_hierarchy(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.task.get("external_id") or self.task.get("id"),
|
|
"parent_id": self.task.get("parent_external_id"),
|
|
"root_id": self.task.get("root_external_id") or self.task.get("external_id") or self.task.get("id"),
|
|
"depth": self.task.get("depth") or 0,
|
|
"level": self.task.get("task_level") or self.payload.get("level") or "",
|
|
}
|
|
|
|
def _observation_summary(self, observation: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"url": observation.get("url"),
|
|
"final_url": observation.get("final_url"),
|
|
"http_status": observation.get("http_status"),
|
|
"content_type": observation.get("content_type"),
|
|
"page_title": observation.get("page_title"),
|
|
"requires_auth": observation.get("requires_auth"),
|
|
"auth_challenge": observation.get("auth_challenge"),
|
|
"has_login_form": observation.get("has_login_form"),
|
|
"link_count": len(observation.get("links") or []),
|
|
"form_count": len(observation.get("forms") or []),
|
|
"button_count": len(observation.get("buttons") or []),
|
|
}
|
|
|
|
def _child_task_briefs(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"external_id": child.get("external_id"),
|
|
"title": child.get("title"),
|
|
"parent_external_id": child.get("parent_external_id"),
|
|
"root_external_id": child.get("root_external_id"),
|
|
"depth": child.get("depth"),
|
|
"task_level": child.get("task_level"),
|
|
"risk_level": child.get("risk_level"),
|
|
"target_url": (child.get("payload") or {}).get("target_url"),
|
|
"source": (child.get("payload") or {}).get("source"),
|
|
"preconditions": (child.get("payload") or {}).get("preconditions") or [],
|
|
"llm_reason": (child.get("payload") or {}).get("llm_reason") or "",
|
|
}
|
|
for child in children
|
|
]
|
|
|
|
def _model_duration(self, host_decision: dict[str, Any], app_decision: dict[str, Any]) -> float:
|
|
return round(
|
|
float(host_decision.get("model_duration_seconds") or 0)
|
|
+ float(app_decision.get("model_duration_seconds") or 0),
|
|
3,
|
|
)
|
|
|
|
def _model_errors(self, *parts: dict[str, Any]) -> list[str]:
|
|
errors: list[str] = []
|
|
for part in parts:
|
|
error = str(part.get("model_error") or part.get("error") or "").strip()
|
|
if error and error not in errors:
|
|
errors.append(error)
|
|
return errors
|