Add SystemTest Agent and Controller
This commit is contained in:
Codex committed
commit
53f523f187
31 files changed
+9150
No files matched your search
@@ -0,0 +1,848 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from .memory import Blackboard
|
||||
from .model_client import ModelClient
|
||||
from .prompts import APP_AGENT_SYSTEM_PROMPT, HOST_AGENT_SYSTEM_PROMPT, SYSTEM_TEST_ANALYST_PROMPT
|
||||
from .tools import AgentToolRegistry, ToolResult, normalize_url
|
||||
|
||||
|
||||
SubmitTask = Callable[[dict[str, Any]], dict[str, Any]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentSessionResult:
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
class AgentSession:
|
||||
"""
|
||||
Minimal UFO-inspired execution loop for one claimed task.
|
||||
|
||||
The implementation keeps the same conceptual boundaries as UFO:
|
||||
HostAgent decides the subtask, AppAgent performs one safe step, Blackboard
|
||||
persists requests/observations/trajectories for later rounds.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
task: dict[str, Any],
|
||||
model_config_state: dict[str, Any],
|
||||
memory_path: Path,
|
||||
request_timeout_seconds: float,
|
||||
submit_task: SubmitTask | None = None,
|
||||
) -> None:
|
||||
self.task = task
|
||||
self.model_config_state = model_config_state
|
||||
self.model_config = model_config_state.get("value") if isinstance(model_config_state.get("value"), dict) else {}
|
||||
self.request_timeout_seconds = request_timeout_seconds
|
||||
self.submit_task = submit_task
|
||||
self.blackboard = Blackboard(memory_path)
|
||||
self.tools = AgentToolRegistry()
|
||||
self.model = ModelClient(self.model_config, timeout_seconds=self._model_timeout_seconds())
|
||||
self.logs: list[str] = []
|
||||
self.screenshot_dir = memory_path.parent / "screenshots"
|
||||
|
||||
def _model_timeout_seconds(self) -> float:
|
||||
try:
|
||||
timeout = float(self.model_config.get("request_timeout_seconds") or self.request_timeout_seconds)
|
||||
except (TypeError, ValueError):
|
||||
timeout = self.request_timeout_seconds
|
||||
return max(1.0, min(timeout, 300.0))
|
||||
|
||||
@property
|
||||
def payload(self) -> dict[str, Any]:
|
||||
value = self.task.get("payload")
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
@property
|
||||
def task_id(self) -> str:
|
||||
return str(self.task.get("external_id") or self.task.get("id") or "")
|
||||
|
||||
def target_url(self) -> str:
|
||||
return normalize_url(
|
||||
self.payload.get("target_url")
|
||||
or self.payload.get("url")
|
||||
or self.payload.get("site_url")
|
||||
or self.task.get("target_url")
|
||||
or self.model_config.get("target_system_url")
|
||||
)
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
started = time.monotonic()
|
||||
self.blackboard.append(
|
||||
"requests",
|
||||
{
|
||||
"task": self._task_brief(),
|
||||
"model_config_version": self.model_config_state.get("version"),
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
|
||||
host_decision = self._host_decision()
|
||||
if self.execution_mode() == "gui":
|
||||
return self._run_gui_task(started, host_decision)
|
||||
|
||||
target_url = self.target_url()
|
||||
if not target_url:
|
||||
return self._finish_without_target(started, host_decision)
|
||||
|
||||
open_browser_result: ToolResult | None = None
|
||||
if self.payload.get("open_in_browser", True):
|
||||
open_browser_result = self.tools.web.open_in_browser(
|
||||
target_url,
|
||||
wait_seconds=float(self.payload.get("browser_wait_seconds") or 1.5),
|
||||
)
|
||||
self.blackboard.append(
|
||||
"actions",
|
||||
{
|
||||
"tool": "open_in_browser",
|
||||
"ok": open_browser_result.ok,
|
||||
"error": open_browser_result.error,
|
||||
"result": open_browser_result.result,
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
if open_browser_result.ok:
|
||||
self.logs.append(f"Opened target URL in browser: {target_url}")
|
||||
else:
|
||||
self.logs.append(f"Browser open failed: {open_browser_result.error}")
|
||||
|
||||
observation_result = self.tools.web.observe_page(
|
||||
target_url,
|
||||
timeout_seconds=self.request_timeout_seconds,
|
||||
)
|
||||
observation = observation_result.result
|
||||
desktop_screenshot_result: ToolResult | None = None
|
||||
if self.payload.get("capture_screenshot", True):
|
||||
desktop_screenshot_result = self.tools.gui.capture_desktop(
|
||||
self.screenshot_dir,
|
||||
prefix=f"task_{self.task.get('id') or self.task_id}_web",
|
||||
)
|
||||
if desktop_screenshot_result.ok:
|
||||
self.blackboard.append(
|
||||
"screenshots",
|
||||
{
|
||||
"image_path": desktop_screenshot_result.result.get("image_path"),
|
||||
"scope": "web_desktop",
|
||||
"target_url": target_url,
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
elif desktop_screenshot_result.error:
|
||||
self.logs.append(f"Desktop screenshot failed: {desktop_screenshot_result.error}")
|
||||
|
||||
self.blackboard.append(
|
||||
"observations",
|
||||
{
|
||||
"tool": "observe_web_page",
|
||||
"ok": observation_result.ok,
|
||||
"error": observation_result.error,
|
||||
"browser_open": open_browser_result.result if open_browser_result else None,
|
||||
"screenshot": desktop_screenshot_result.result if desktop_screenshot_result and desktop_screenshot_result.ok else None,
|
||||
"summary": self._observation_summary(observation),
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
|
||||
app_decision = self._app_decision(host_decision, observation_result)
|
||||
discovered_children: list[dict[str, Any]] = []
|
||||
model_generated_children: list[dict[str, Any]] = []
|
||||
rule_discovered_children: list[dict[str, Any]] = []
|
||||
submitted_children: list[dict[str, Any]] = []
|
||||
model_analysis: dict[str, Any] = {
|
||||
"model_used": False,
|
||||
"model_available": self.model.available,
|
||||
}
|
||||
discovery_source = "none"
|
||||
if self._allow_child_discovery(observation_result):
|
||||
model_analysis, model_generated_children = self._model_discover_child_tasks(observation_result)
|
||||
if model_generated_children:
|
||||
discovered_children = model_generated_children
|
||||
discovery_source = "llm"
|
||||
else:
|
||||
rule_discovered_children = self.tools.web.discover_child_tasks(
|
||||
self.task,
|
||||
observation,
|
||||
max_depth=int(self.payload.get("discovery_max_depth") or 2),
|
||||
)
|
||||
discovered_children = rule_discovered_children
|
||||
discovery_source = "rule"
|
||||
if model_analysis.get("model_available") and model_analysis.get("model_used"):
|
||||
self.logs.append("Model returned no usable functional tasks; used rule-based fallback.")
|
||||
submitted_children = self._submit_child_tasks(discovered_children)
|
||||
|
||||
success, message, needs_user_input = self._status_from_observation(observation_result, app_decision)
|
||||
if model_analysis.get("needs_user_input"):
|
||||
needs_user_input = True
|
||||
success = False
|
||||
message = str(model_analysis.get("user_input_reason") or "Model determined that user input is required.")
|
||||
duration = round(time.monotonic() - started, 3)
|
||||
result = {
|
||||
"success": success,
|
||||
"message": message,
|
||||
"task_type": self.payload.get("task_type") or "manual",
|
||||
"agent_framework": "ufo-inspired-lite",
|
||||
"p0_mode": True,
|
||||
"needs_user_input": needs_user_input,
|
||||
"target_url": target_url,
|
||||
"final_url": observation.get("final_url"),
|
||||
"http_status": observation.get("http_status"),
|
||||
"content_type": observation.get("content_type"),
|
||||
"page_title": observation.get("page_title"),
|
||||
"duration_seconds": duration,
|
||||
"action_duration_seconds": round(
|
||||
observation_result.duration_seconds
|
||||
+ (open_browser_result.duration_seconds if open_browser_result else 0)
|
||||
+ (desktop_screenshot_result.duration_seconds if desktop_screenshot_result else 0),
|
||||
3,
|
||||
),
|
||||
"model_duration_seconds": round(
|
||||
self._model_duration(host_decision, app_decision)
|
||||
+ float(model_analysis.get("model_duration_seconds") or 0),
|
||||
3,
|
||||
),
|
||||
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
||||
"host_decision": host_decision,
|
||||
"app_decision": app_decision,
|
||||
"model_available": self.model.available,
|
||||
"model_analysis": model_analysis,
|
||||
"model_errors": self._model_errors(host_decision, app_decision, model_analysis),
|
||||
"observation": self._observation_summary(observation),
|
||||
"browser_open": open_browser_result.result if open_browser_result else None,
|
||||
"screenshots": [
|
||||
desktop_screenshot_result.result
|
||||
] if desktop_screenshot_result and desktop_screenshot_result.ok else [],
|
||||
"available_tools": self.tools.descriptions(),
|
||||
"discovered_child_tasks": self._child_task_briefs(discovered_children),
|
||||
"model_generated_child_tasks": self._child_task_briefs(model_generated_children),
|
||||
"rule_discovered_child_tasks": self._child_task_briefs(rule_discovered_children),
|
||||
"discovery_source": discovery_source,
|
||||
"submitted_child_tasks": submitted_children,
|
||||
"memory": self.blackboard.summary(),
|
||||
"logs": [*self.logs, *(observation_result.result.get("logs") or [])],
|
||||
}
|
||||
self.blackboard.append(
|
||||
"trajectories",
|
||||
{
|
||||
"host_decision": host_decision,
|
||||
"app_decision": app_decision,
|
||||
"result": {
|
||||
"success": success,
|
||||
"message": message,
|
||||
"submitted_child_count": len(submitted_children),
|
||||
"duration_seconds": duration,
|
||||
},
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
result["memory"] = self.blackboard.summary()
|
||||
return result
|
||||
|
||||
def execution_mode(self) -> str:
|
||||
explicit = str(self.payload.get("execution_mode") or self.payload.get("agent_mode") or "").strip().lower()
|
||||
if explicit in {"web", "gui"}:
|
||||
return explicit
|
||||
task_type = str(self.payload.get("task_type") or "").lower()
|
||||
if task_type.startswith("gui") or self.payload.get("window_id") or self.payload.get("window_title"):
|
||||
return "gui"
|
||||
return "web"
|
||||
|
||||
def _run_gui_task(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]:
|
||||
windows_result = self.tools.gui.list_windows()
|
||||
screenshot_result = self.tools.gui.capture_desktop(self.screenshot_dir, prefix=f"task_{self.task.get('id') or self.task_id}_desktop")
|
||||
selected_window = self._select_window(windows_result.result.get("windows") or [])
|
||||
inspect_result: ToolResult | None = None
|
||||
window_screenshot_result: ToolResult | None = None
|
||||
if selected_window:
|
||||
inspect_result = self.tools.gui.inspect_window(str(selected_window["id"]))
|
||||
window_screenshot_result = self.tools.gui.capture_window(
|
||||
str(selected_window["id"]),
|
||||
self.screenshot_dir,
|
||||
prefix=f"task_{self.task.get('id') or self.task_id}_window",
|
||||
)
|
||||
|
||||
observation = self._gui_observation_summary(
|
||||
windows_result,
|
||||
screenshot_result,
|
||||
selected_window,
|
||||
inspect_result,
|
||||
window_screenshot_result,
|
||||
)
|
||||
self.blackboard.append(
|
||||
"observations",
|
||||
{
|
||||
"tool": "gui_observe",
|
||||
"ok": windows_result.ok,
|
||||
"error": windows_result.error,
|
||||
"summary": observation,
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
if screenshot_result.ok:
|
||||
self.blackboard.append(
|
||||
"screenshots",
|
||||
{"image_path": screenshot_result.result.get("image_path"), "scope": "desktop"},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
if window_screenshot_result and window_screenshot_result.ok:
|
||||
self.blackboard.append(
|
||||
"screenshots",
|
||||
{"image_path": window_screenshot_result.result.get("image_path"), "scope": "window", "window": selected_window},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
|
||||
app_decision = {
|
||||
"observation": json.dumps(observation, ensure_ascii=False),
|
||||
"thought": "GUI P0 阶段只枚举窗口、控件并保存证据截图，不执行点击或输入。",
|
||||
"action": {
|
||||
"function": "list_windows" if not selected_window else "inspect_window_controls",
|
||||
"arguments": {
|
||||
"window_id": selected_window.get("id") if selected_window else None,
|
||||
"window_title": self.payload.get("window_title"),
|
||||
},
|
||||
"status": "FINISH" if windows_result.ok and (not self.payload.get("window_title") or selected_window) else "FAIL",
|
||||
},
|
||||
"plan": [],
|
||||
"comment": "GUI 观察完成。" if windows_result.ok else windows_result.error,
|
||||
"save_screenshot": {
|
||||
"save": bool(screenshot_result.ok or (window_screenshot_result and window_screenshot_result.ok)),
|
||||
"reason": "GUI P0 观察需要截图作为证据。" if screenshot_result.ok else screenshot_result.error,
|
||||
},
|
||||
"result": "GUI 窗口和控件信息已记录。" if windows_result.ok else windows_result.error,
|
||||
"model_used": False,
|
||||
}
|
||||
|
||||
success = windows_result.ok and (not self.payload.get("window_title") or bool(selected_window))
|
||||
message = "Agent GUI P0 observation completed." if success else "Requested GUI window was not found or could not be inspected."
|
||||
duration = round(time.monotonic() - started, 3)
|
||||
result = {
|
||||
"success": success,
|
||||
"message": message,
|
||||
"task_type": self.payload.get("task_type") or "gui_p0",
|
||||
"agent_framework": "ufo-inspired-lite",
|
||||
"p0_mode": True,
|
||||
"execution_mode": "gui",
|
||||
"needs_user_input": False,
|
||||
"duration_seconds": duration,
|
||||
"action_duration_seconds": round(
|
||||
windows_result.duration_seconds
|
||||
+ screenshot_result.duration_seconds
|
||||
+ (inspect_result.duration_seconds if inspect_result else 0)
|
||||
+ (window_screenshot_result.duration_seconds if window_screenshot_result else 0),
|
||||
3,
|
||||
),
|
||||
"model_duration_seconds": self._model_duration(host_decision, app_decision),
|
||||
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
||||
"host_decision": host_decision,
|
||||
"app_decision": app_decision,
|
||||
"observation": observation,
|
||||
"available_tools": self.tools.descriptions(),
|
||||
"memory": self.blackboard.summary(),
|
||||
"logs": [
|
||||
*self.logs,
|
||||
*(windows_result.error and [windows_result.error] or []),
|
||||
*(screenshot_result.error and [screenshot_result.error] or []),
|
||||
*(inspect_result and inspect_result.error and [inspect_result.error] or []),
|
||||
],
|
||||
}
|
||||
self.blackboard.append(
|
||||
"trajectories",
|
||||
{
|
||||
"host_decision": host_decision,
|
||||
"app_decision": app_decision,
|
||||
"result": {
|
||||
"success": success,
|
||||
"message": message,
|
||||
"duration_seconds": duration,
|
||||
},
|
||||
},
|
||||
task_id=self.task_id,
|
||||
)
|
||||
result["memory"] = self.blackboard.summary()
|
||||
return result
|
||||
|
||||
def _host_decision(self) -> dict[str, Any]:
|
||||
fallback = {
|
||||
"observation": "收到一个主控端分配的测试任务。",
|
||||
"thought": "P0 阶段先把任务限定到单个 Web/AppAgent，执行无副作用观察，并按页面结构生成后续子任务。",
|
||||
"current_subtask": self.payload.get("objective") or self.payload.get("instruction") or self.task.get("title") or "",
|
||||
"message": ["只执行无副作用观察动作；遇到登录、验证码、权限或破坏性入口时暂停并回传。"],
|
||||
"status": "ASSIGN",
|
||||
"plan": ["观察目标页面", "记录页面元信息", "识别可继续拆分的链接、表单和入口"],
|
||||
"function": "assign_app_agent",
|
||||
"arguments": {"agent": "web_app_agent", "target_url": self.target_url()},
|
||||
"questions": [],
|
||||
"result": "已分配给 Web AppAgent 执行 P0 观察。",
|
||||
"model_used": False,
|
||||
}
|
||||
if not self.model.available:
|
||||
return fallback
|
||||
|
||||
context = {
|
||||
"task": self._task_brief(),
|
||||
"blackboard": self.blackboard.snapshot(limit=5),
|
||||
"tools": self.tools.descriptions(),
|
||||
}
|
||||
response = self.model.chat_json(
|
||||
[
|
||||
{"role": "system", "content": HOST_AGENT_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
|
||||
]
|
||||
)
|
||||
if response.ok and isinstance(response.parsed, dict):
|
||||
parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds}
|
||||
return parsed
|
||||
self.logs.append(f"HostAgent model fallback: {response.error or 'invalid JSON response'}")
|
||||
return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds}
|
||||
|
||||
def _app_decision(self, host_decision: dict[str, Any], observation_result: ToolResult) -> dict[str, Any]:
|
||||
observation = self._observation_summary(observation_result.result)
|
||||
fallback_status = "FINISH" if observation_result.ok else "FAIL"
|
||||
if observation.get("requires_auth"):
|
||||
fallback_status = "PENDING"
|
||||
fallback = {
|
||||
"observation": json.dumps(observation, ensure_ascii=False),
|
||||
"thought": "P0 阶段采用 observe_web_page 工具完成可达性和页面结构观察，不执行提交、保存、删除等有副作用动作。",
|
||||
"action": {
|
||||
"function": "observe_web_page",
|
||||
"arguments": {"target_url": self.target_url()},
|
||||
"status": fallback_status,
|
||||
},
|
||||
"plan": [],
|
||||
"comment": "页面观察完成。" if observation_result.ok else observation_result.error,
|
||||
"save_screenshot": {"save": False, "reason": "P0 Web 观察暂不保存截图。"},
|
||||
"result": "页面可达性和结构信息已记录。" if observation_result.ok else observation_result.error,
|
||||
"model_used": False,
|
||||
}
|
||||
if not self.model.available:
|
||||
return fallback
|
||||
|
||||
context = {
|
||||
"host_decision": host_decision,
|
||||
"task": self._task_brief(),
|
||||
"observation": observation,
|
||||
"available_tools": self.tools.descriptions(),
|
||||
"recent_trajectories": self.blackboard.recent("trajectories", 5),
|
||||
}
|
||||
response = self.model.chat_json(
|
||||
[
|
||||
{"role": "system", "content": APP_AGENT_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
|
||||
]
|
||||
)
|
||||
if response.ok and isinstance(response.parsed, dict):
|
||||
parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds}
|
||||
return parsed
|
||||
self.logs.append(f"AppAgent model fallback: {response.error or 'invalid JSON response'}")
|
||||
return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds}
|
||||
|
||||
def _model_discover_child_tasks(self, observation_result: ToolResult) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
base_analysis: dict[str, Any] = {
|
||||
"model_available": self.model.available,
|
||||
"model_called": False,
|
||||
"model_call_succeeded": False,
|
||||
"model_used": False,
|
||||
"model_duration_seconds": 0,
|
||||
"summary": "",
|
||||
"coverage_notes": [],
|
||||
"needs_user_input": False,
|
||||
"user_input_reason": "",
|
||||
"functional_task_count": 0,
|
||||
}
|
||||
if not observation_result.ok:
|
||||
return {**base_analysis, "error": observation_result.error or "Page observation failed."}, []
|
||||
if not self.model.available:
|
||||
self.logs.append("Model config is incomplete; used rule-based discovery fallback.")
|
||||
return {**base_analysis, "error": "Model config is incomplete."}, []
|
||||
|
||||
observation = observation_result.result
|
||||
context = {
|
||||
"task": self._task_brief(),
|
||||
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
||||
"page": self._observation_for_model(observation),
|
||||
"recent_memory": self.blackboard.snapshot(limit=4),
|
||||
"constraints": {
|
||||
"p0_no_side_effects": True,
|
||||
"max_functional_tasks": 8,
|
||||
"discovery_max_depth": int(self.payload.get("discovery_max_depth") or 2),
|
||||
},
|
||||
}
|
||||
response = self.model.chat_json(
|
||||
[
|
||||
{"role": "system", "content": SYSTEM_TEST_ANALYST_PROMPT},
|
||||
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
|
||||
]
|
||||
)
|
||||
if not response.ok or not isinstance(response.parsed, dict):
|
||||
self.logs.append(f"SystemTestAnalyst model fallback: {response.error or 'invalid JSON response'}")
|
||||
return {
|
||||
**base_analysis,
|
||||
"model_called": True,
|
||||
"model_call_succeeded": False,
|
||||
"model_used": False,
|
||||
"model_duration_seconds": response.duration_seconds,
|
||||
"error": response.error or "Model did not return a valid JSON object.",
|
||||
}, []
|
||||
|
||||
parsed = response.parsed
|
||||
children = self._children_from_model_analysis(parsed, observation)
|
||||
analysis = {
|
||||
**base_analysis,
|
||||
"model_called": True,
|
||||
"model_call_succeeded": True,
|
||||
"model_used": True,
|
||||
"model_duration_seconds": response.duration_seconds,
|
||||
"summary": str(parsed.get("summary") or ""),
|
||||
"coverage_notes": parsed.get("coverage_notes") if isinstance(parsed.get("coverage_notes"), list) else [],
|
||||
"needs_user_input": bool(parsed.get("needs_user_input")),
|
||||
"user_input_reason": str(parsed.get("user_input_reason") or ""),
|
||||
"functional_task_count": len(children),
|
||||
}
|
||||
self.logs.append(
|
||||
"Model analyzed page and generated "
|
||||
f"{len(children)} functional task(s): {analysis.get('summary') or 'no summary'}"
|
||||
)
|
||||
return analysis, children
|
||||
|
||||
def _observation_for_model(self, observation: dict[str, Any]) -> dict[str, Any]:
|
||||
def clean_items(items: Any, limit: int) -> list[dict[str, Any]]:
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
cleaned: list[dict[str, Any]] = []
|
||||
for item in items[:limit]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
cleaned.append(
|
||||
{
|
||||
key: value
|
||||
for key, value in item.items()
|
||||
if key in {"href", "text", "action", "method", "id", "name", "class", "type", "fields", "placeholder", "aria_label", "in_form"}
|
||||
}
|
||||
)
|
||||
return cleaned
|
||||
|
||||
return {
|
||||
"url": observation.get("url"),
|
||||
"final_url": observation.get("final_url"),
|
||||
"http_status": observation.get("http_status"),
|
||||
"content_type": observation.get("content_type"),
|
||||
"page_title": observation.get("page_title"),
|
||||
"requires_auth": observation.get("requires_auth"),
|
||||
"has_login_form": observation.get("has_login_form"),
|
||||
"links": clean_items(observation.get("links"), 20),
|
||||
"forms": clean_items(observation.get("forms"), 12),
|
||||
"buttons": clean_items(observation.get("buttons"), 20),
|
||||
}
|
||||
|
||||
def _children_from_model_analysis(self, analysis: dict[str, Any], observation: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw_items = analysis.get("functional_tasks")
|
||||
if not isinstance(raw_items, list):
|
||||
return []
|
||||
max_children = 8
|
||||
children: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for index, item in enumerate(raw_items[:max_children], start=1):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
child = self._child_from_model_item(item, observation, index)
|
||||
key = (
|
||||
str(child.get("title") or "").strip().lower(),
|
||||
str((child.get("payload") or {}).get("target_url") or "").strip().lower(),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
children.append(child)
|
||||
return children
|
||||
|
||||
def _child_from_model_item(self, item: dict[str, Any], observation: dict[str, Any], index: int) -> dict[str, Any]:
|
||||
parent_external_id = str(self.task.get("external_id") or self.task.get("id") or "")
|
||||
root_external_id = str(self.task.get("root_external_id") or self.payload.get("root_external_id") or parent_external_id)
|
||||
parent_depth = int(self.task.get("depth") or self.payload.get("depth") or 0)
|
||||
depth = parent_depth + 1
|
||||
discovery_max_depth = int(self.payload.get("discovery_max_depth") or 2)
|
||||
base_url = observation.get("final_url") or observation.get("url") or self.target_url()
|
||||
raw_target_url = str(item.get("target_url") or "").strip()
|
||||
target_url = urljoin(base_url, raw_target_url) if raw_target_url else base_url
|
||||
title = str(item.get("title") or f"模型生成测试任务 {index}").strip()
|
||||
objective = str(item.get("objective") or title).strip()
|
||||
module = str(item.get("module") or title).strip()
|
||||
level = str(item.get("level") or "feature").strip().lower()
|
||||
if level not in {"module", "feature", "case"}:
|
||||
level = "feature"
|
||||
risk = str(item.get("risk_level") or "low").strip().lower()
|
||||
if risk not in {"low", "medium", "high"}:
|
||||
risk = "low"
|
||||
try:
|
||||
priority = int(item.get("priority") or 30 + depth * 10 + index)
|
||||
except (TypeError, ValueError):
|
||||
priority = 30 + depth * 10 + index
|
||||
preconditions = item.get("preconditions") if isinstance(item.get("preconditions"), list) else []
|
||||
external_id = self._stable_child_id(root_external_id, parent_external_id, title, target_url, objective)
|
||||
payload = {
|
||||
"task_type": "smart_discovery_p0",
|
||||
"instruction": objective,
|
||||
"objective": objective,
|
||||
"target_url": target_url,
|
||||
"source": "llm_analysis",
|
||||
"llm_reason": item.get("reason") or "",
|
||||
"level": level,
|
||||
"depth": depth,
|
||||
"risk_level": risk,
|
||||
"allow_child_discovery": level in {"module", "feature"} and depth < discovery_max_depth,
|
||||
"discovery_max_depth": discovery_max_depth,
|
||||
"open_in_browser": True,
|
||||
"capture_screenshot": True,
|
||||
"preconditions": preconditions,
|
||||
"hierarchy": {
|
||||
"id": external_id,
|
||||
"parent_id": parent_external_id,
|
||||
"root_id": root_external_id,
|
||||
"depth": depth,
|
||||
"level": level,
|
||||
},
|
||||
}
|
||||
return {
|
||||
"external_id": external_id,
|
||||
"title": title,
|
||||
"module": module,
|
||||
"parent_external_id": parent_external_id,
|
||||
"root_external_id": root_external_id,
|
||||
"depth": depth,
|
||||
"task_level": level,
|
||||
"risk_level": risk,
|
||||
"priority": priority,
|
||||
"failure_policy": "pause" if risk in {"medium", "high"} or preconditions else "skip",
|
||||
"max_retries": 2,
|
||||
"timeout_seconds": 90,
|
||||
"tags": ["llm-generated", "system-test", "p0"],
|
||||
"payload": payload,
|
||||
}
|
||||
|
||||
def _stable_child_id(self, *parts: Any) -> str:
|
||||
raw = "|".join(str(part or "") for part in parts)
|
||||
return f"llm-{hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]}"
|
||||
|
||||
def _allow_child_discovery(self, observation_result: ToolResult) -> bool:
|
||||
if not observation_result.ok:
|
||||
return False
|
||||
allow = self.payload.get("allow_child_discovery")
|
||||
if allow is not None:
|
||||
return bool(allow)
|
||||
if self.payload.get("task_type") != "smart_discovery_p0":
|
||||
return False
|
||||
level = str(self.payload.get("level") or self.task.get("task_level") or "")
|
||||
depth = int(self.payload.get("depth") or self.task.get("depth") or 0)
|
||||
return level in {"site", "module", ""} and depth < int(self.payload.get("discovery_max_depth") or 2)
|
||||
|
||||
def _submit_child_tasks(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
if not children or self.submit_task is None:
|
||||
return []
|
||||
submitted: list[dict[str, Any]] = []
|
||||
plan_name = self.task.get("plan_name") or self.payload.get("plan_name") or "智能探索 P0"
|
||||
for child in children:
|
||||
payload = {**child, "plan_name": plan_name}
|
||||
try:
|
||||
created = self.submit_task(payload)
|
||||
except Exception as exc:
|
||||
self.logs.append(f"Submit child task failed: {child.get('external_id')} error={exc}")
|
||||
continue
|
||||
submitted.append(
|
||||
{
|
||||
"external_id": created.get("external_id") or child.get("external_id"),
|
||||
"title": created.get("title") or child.get("title"),
|
||||
"depth": created.get("depth") if created.get("depth") is not None else child.get("depth"),
|
||||
"task_level": created.get("task_level") or child.get("task_level"),
|
||||
}
|
||||
)
|
||||
return submitted
|
||||
|
||||
def _status_from_observation(self, observation_result: ToolResult, app_decision: dict[str, Any]) -> tuple[bool, str, bool]:
|
||||
action = app_decision.get("action") if isinstance(app_decision.get("action"), dict) else {}
|
||||
status = str(action.get("status") or "").upper()
|
||||
if status == "PENDING":
|
||||
return False, "Task needs user input before it can continue.", True
|
||||
if status in {"FAIL", "CONFIRM"}:
|
||||
return False, app_decision.get("result") or app_decision.get("comment") or "Agent action did not complete.", status == "CONFIRM"
|
||||
if not observation_result.ok:
|
||||
return False, observation_result.error or "Target observation failed.", False
|
||||
if observation_result.result.get("http_status") in {401, 403}:
|
||||
return False, "Target page requires authentication or permission.", True
|
||||
return True, "Agent P0 observation completed.", False
|
||||
|
||||
def _finish_without_target(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]:
|
||||
task_type = self.payload.get("task_type") or "manual"
|
||||
success = task_type != "smart_discovery_p0"
|
||||
duration = round(time.monotonic() - started, 3)
|
||||
result = {
|
||||
"success": success,
|
||||
"message": "No executable target URL was provided." if not success else "Task accepted; no executable URL was provided for P0 mode.",
|
||||
"task_type": task_type,
|
||||
"agent_framework": "ufo-inspired-lite",
|
||||
"p0_mode": True,
|
||||
"needs_user_input": not success,
|
||||
"duration_seconds": duration,
|
||||
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
|
||||
"host_decision": host_decision,
|
||||
"app_decision": {
|
||||
"action": {"function": "no_action", "arguments": {}, "status": "FAIL" if not success else "FINISH"},
|
||||
"result": "Missing target URL.",
|
||||
},
|
||||
"available_tools": self.tools.descriptions(),
|
||||
"memory": self.blackboard.summary(),
|
||||
"logs": ["AgentSession requires target_url for Web P0 execution."],
|
||||
}
|
||||
self.blackboard.append("trajectories", {"host_decision": host_decision, "result": result}, task_id=self.task_id)
|
||||
return result
|
||||
|
||||
def _select_window(self, windows: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
window_id = self.payload.get("window_id")
|
||||
if window_id:
|
||||
for window in windows:
|
||||
if str(window.get("id")) == str(window_id) or str(window.get("hex_id")).lower() == str(window_id).lower():
|
||||
return window
|
||||
return None
|
||||
|
||||
title = str(self.payload.get("window_title") or self.payload.get("application") or "").strip().lower()
|
||||
if not title:
|
||||
return None
|
||||
for window in windows:
|
||||
if title in str(window.get("title") or "").lower():
|
||||
return window
|
||||
for window in windows:
|
||||
if title in str(window.get("class_name") or "").lower():
|
||||
return window
|
||||
return None
|
||||
|
||||
def _gui_observation_summary(
|
||||
self,
|
||||
windows_result: ToolResult,
|
||||
screenshot_result: ToolResult,
|
||||
selected_window: dict[str, Any] | None,
|
||||
inspect_result: ToolResult | None,
|
||||
window_screenshot_result: ToolResult | None,
|
||||
) -> dict[str, Any]:
|
||||
windows = windows_result.result.get("windows") if isinstance(windows_result.result, dict) else []
|
||||
controls = []
|
||||
if inspect_result and isinstance(inspect_result.result, dict):
|
||||
controls = inspect_result.result.get("controls") or []
|
||||
return {
|
||||
"backend": (windows_result.result or {}).get("backend") or self.tools.gui.backend_status(),
|
||||
"window_count": len(windows or []),
|
||||
"windows": [
|
||||
{
|
||||
"id": window.get("id"),
|
||||
"title": window.get("title"),
|
||||
"class_name": window.get("class_name"),
|
||||
"process_id": window.get("process_id"),
|
||||
"rect": window.get("rect"),
|
||||
"minimized": window.get("minimized"),
|
||||
}
|
||||
for window in (windows or [])[:20]
|
||||
],
|
||||
"selected_window": selected_window,
|
||||
"control_count": len(controls),
|
||||
"controls": [
|
||||
{
|
||||
"id": control.get("id"),
|
||||
"text": control.get("text"),
|
||||
"class_name": control.get("class_name"),
|
||||
"rect": control.get("rect"),
|
||||
"visible": control.get("visible"),
|
||||
"enabled": control.get("enabled"),
|
||||
}
|
||||
for control in controls[:80]
|
||||
],
|
||||
"desktop_screenshot": screenshot_result.result if screenshot_result.ok else None,
|
||||
"window_screenshot": window_screenshot_result.result if window_screenshot_result and window_screenshot_result.ok else None,
|
||||
"errors": [
|
||||
message
|
||||
for message in [
|
||||
windows_result.error,
|
||||
screenshot_result.error,
|
||||
inspect_result.error if inspect_result else "",
|
||||
window_screenshot_result.error if window_screenshot_result else "",
|
||||
]
|
||||
if message
|
||||
],
|
||||
}
|
||||
|
||||
def _task_brief(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.task.get("id"),
|
||||
"external_id": self.task.get("external_id"),
|
||||
"title": self.task.get("title"),
|
||||
"plan_name": self.task.get("plan_name"),
|
||||
"module": self.task.get("module"),
|
||||
"status": self.task.get("status"),
|
||||
"depth": self.task.get("depth"),
|
||||
"task_level": self.task.get("task_level"),
|
||||
"risk_level": self.task.get("risk_level"),
|
||||
"payload": self.payload,
|
||||
}
|
||||
|
||||
def _task_hierarchy(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.task.get("external_id") or self.task.get("id"),
|
||||
"parent_id": self.task.get("parent_external_id"),
|
||||
"root_id": self.task.get("root_external_id") or self.task.get("external_id") or self.task.get("id"),
|
||||
"depth": self.task.get("depth") or 0,
|
||||
"level": self.task.get("task_level") or self.payload.get("level") or "",
|
||||
}
|
||||
|
||||
def _observation_summary(self, observation: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"url": observation.get("url"),
|
||||
"final_url": observation.get("final_url"),
|
||||
"http_status": observation.get("http_status"),
|
||||
"content_type": observation.get("content_type"),
|
||||
"page_title": observation.get("page_title"),
|
||||
"requires_auth": observation.get("requires_auth"),
|
||||
"auth_challenge": observation.get("auth_challenge"),
|
||||
"has_login_form": observation.get("has_login_form"),
|
||||
"link_count": len(observation.get("links") or []),
|
||||
"form_count": len(observation.get("forms") or []),
|
||||
"button_count": len(observation.get("buttons") or []),
|
||||
}
|
||||
|
||||
def _child_task_briefs(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"external_id": child.get("external_id"),
|
||||
"title": child.get("title"),
|
||||
"parent_external_id": child.get("parent_external_id"),
|
||||
"root_external_id": child.get("root_external_id"),
|
||||
"depth": child.get("depth"),
|
||||
"task_level": child.get("task_level"),
|
||||
"risk_level": child.get("risk_level"),
|
||||
"target_url": (child.get("payload") or {}).get("target_url"),
|
||||
"source": (child.get("payload") or {}).get("source"),
|
||||
"preconditions": (child.get("payload") or {}).get("preconditions") or [],
|
||||
"llm_reason": (child.get("payload") or {}).get("llm_reason") or "",
|
||||
}
|
||||
for child in children
|
||||
]
|
||||
|
||||
def _model_duration(self, host_decision: dict[str, Any], app_decision: dict[str, Any]) -> float:
|
||||
return round(
|
||||
float(host_decision.get("model_duration_seconds") or 0)
|
||||
+ float(app_decision.get("model_duration_seconds") or 0),
|
||||
3,
|
||||
)
|
||||
|
||||
def _model_errors(self, *parts: dict[str, Any]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for part in parts:
|
||||
error = str(part.get("model_error") or part.get("error") or "").strip()
|
||||
if error and error not in errors:
|
||||
errors.append(error)
|
||||
return errors
|
||||
Reference in new issue
Block a user