Add SystemTest Agent and Controller

This commit is contained in:
Codex committed 2026-06-22 10:37:20 +08:00
commit 53f523f187
31 files changed
+9150

No files matched your search

@@ -0,0 +1,5 @@
from .memory import Blackboard, MemoryItem
from .session import AgentSession
from .tools import AgentToolRegistry
__all__ = ["AgentSession", "AgentToolRegistry", "Blackboard", "MemoryItem"]
+124
View File
@@ -0,0 +1,124 @@
from __future__ import annotations
import json
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds")
@dataclass
class MemoryItem:
kind: str
content: dict[str, Any]
created_at: str = field(default_factory=utc_now)
task_id: str | None = None
step: int | None = None
metadata: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"kind": self.kind,
"content": self.content,
"created_at": self.created_at,
"task_id": self.task_id,
"step": self.step,
"metadata": self.metadata,
}
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> "MemoryItem":
return cls(
kind=str(payload.get("kind") or "event"),
content=payload.get("content") if isinstance(payload.get("content"), dict) else {},
created_at=str(payload.get("created_at") or utc_now()),
task_id=str(payload.get("task_id")) if payload.get("task_id") is not None else None,
step=int(payload["step"]) if isinstance(payload.get("step"), int) else None,
metadata=payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {},
)
class Blackboard:
"""
Lightweight file-backed memory shared by the local Host/App agent layers.
It intentionally mirrors UFO's blackboard shape without pulling in the full
dependency stack: requests, trajectories, observations, screenshots and facts.
"""
DEFAULT_KINDS = ("requests", "trajectories", "observations", "screenshots", "facts")
def __init__(self, path: Path, max_items_per_kind: int = 200):
self.path = path
self.max_items_per_kind = max_items_per_kind
self._content: dict[str, list[MemoryItem]] = {kind: [] for kind in self.DEFAULT_KINDS}
self.load()
def load(self) -> None:
if not self.path.exists():
return
try:
raw = json.loads(self.path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return
if not isinstance(raw, dict):
return
for kind in self.DEFAULT_KINDS:
entries = raw.get(kind)
if isinstance(entries, list):
self._content[kind] = [
MemoryItem.from_dict(item)
for item in entries
if isinstance(item, dict)
][-self.max_items_per_kind :]
def save(self) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
payload = {
kind: [item.to_dict() for item in items[-self.max_items_per_kind :]]
for kind, items in self._content.items()
}
self.path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
encoding="utf-8",
)
def append(
self,
kind: str,
content: dict[str, Any],
*,
task_id: str | None = None,
step: int | None = None,
metadata: dict[str, Any] | None = None,
) -> MemoryItem:
if kind not in self._content:
self._content[kind] = []
item = MemoryItem(
kind=kind,
content=content,
task_id=task_id,
step=step,
metadata=metadata or {},
)
self._content[kind].append(item)
self._content[kind] = self._content[kind][-self.max_items_per_kind :]
self.save()
return item
def recent(self, kind: str, limit: int = 10) -> list[dict[str, Any]]:
return [item.to_dict() for item in self._content.get(kind, [])[-limit:]]
def snapshot(self, limit: int = 5) -> dict[str, list[dict[str, Any]]]:
return {kind: self.recent(kind, limit) for kind in self._content}
def summary(self) -> dict[str, Any]:
return {
"path": str(self.path),
"counts": {kind: len(items) for kind, items in self._content.items()},
"latest_trajectory": self.recent("trajectories", 1),
}
@@ -0,0 +1,190 @@
from __future__ import annotations
import json
import re
import time
from dataclasses import dataclass
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
@dataclass
class ModelResponse:
ok: bool
content: str = ""
parsed: dict[str, Any] | None = None
duration_seconds: float = 0.0
error: str = ""
def _chat_completions_url(base_url: str) -> str:
base = base_url.rstrip("/")
if base.endswith("/chat/completions"):
return base
if base.endswith("/v1"):
return f"{base}/chat/completions"
return f"{base}/v1/chat/completions"
def parse_json_object(text: str) -> dict[str, Any] | None:
raw = text.strip()
fence = re.search(r"```(?:json)?\s*(.*?)```", raw, re.IGNORECASE | re.DOTALL)
if fence:
raw = fence.group(1).strip()
start = raw.find("{")
end = raw.rfind("}")
if start >= 0 and end > start:
raw = raw[start : end + 1]
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return None
return parsed if isinstance(parsed, dict) else None
def _network_error_message(label: str, exc: BaseException, timeout: float) -> str:
reason = getattr(exc, "reason", exc)
if isinstance(exc, TimeoutError) or isinstance(reason, TimeoutError):
detail = f"timed out after {timeout:g}s"
else:
detail = str(reason) or type(reason).__name__
return f"{label} -> {detail}"
class ModelClient:
def __init__(self, model_config: dict[str, Any], timeout_seconds: float = 30):
self.model_config = model_config
self.timeout_seconds = timeout_seconds
@property
def available(self) -> bool:
return bool(
self.model_config.get("controller_model_proxy_url")
or (self.model_config.get("vllm_base_url") and self.model_config.get("vllm_model"))
)
def chat_json(self, messages: list[dict[str, str]], temperature: float | None = None) -> ModelResponse:
if not self.available:
return ModelResponse(ok=False, error="Model config is incomplete.")
proxy_url = str(self.model_config.get("controller_model_proxy_url") or "").strip()
if proxy_url:
return self._chat_json_via_controller(proxy_url, messages, temperature)
url = _chat_completions_url(str(self.model_config.get("vllm_base_url") or ""))
payload = {
"model": self.model_config.get("vllm_model"),
"messages": messages,
"temperature": temperature if temperature is not None else self.model_config.get("temperature", 0),
}
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
headers = {
"Accept": "application/json",
"Content-Type": "application/json; charset=utf-8",
}
api_key = str(self.model_config.get("api_key") or "")
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
started = time.monotonic()
request = Request(url, data=body, headers=headers, method="POST")
try:
with urlopen(request, timeout=self.timeout_seconds) as response:
raw = response.read()
except HTTPError as exc:
error_body = exc.read().decode("utf-8", errors="replace")
return ModelResponse(
ok=False,
duration_seconds=round(time.monotonic() - started, 3),
error=f"{url} -> HTTP {exc.code}: {error_body or exc.reason}",
)
except (URLError, TimeoutError, OSError) as exc:
return ModelResponse(
ok=False,
duration_seconds=round(time.monotonic() - started, 3),
error=_network_error_message(url, exc, self.timeout_seconds),
)
duration = round(time.monotonic() - started, 3)
try:
data = json.loads(raw.decode("utf-8"))
choice = data["choices"][0]
message = choice.get("message") or {}
content = (
message.get("content")
or message.get("reasoning_content")
or message.get("reasoning")
or choice.get("text")
or ""
)
except (KeyError, IndexError, TypeError, json.JSONDecodeError) as exc:
return ModelResponse(ok=False, duration_seconds=duration, error=f"Invalid model response: {exc}")
if not isinstance(content, str):
content = json.dumps(content, ensure_ascii=False)
return ModelResponse(
ok=True,
content=str(content),
parsed=parse_json_object(str(content)),
duration_seconds=duration,
)
def _chat_json_via_controller(
self,
url: str,
messages: list[dict[str, str]],
temperature: float | None = None,
) -> ModelResponse:
payload = {
"messages": messages,
"temperature": temperature if temperature is not None else self.model_config.get("temperature", 0),
}
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
headers = {
"Accept": "application/json",
"Content-Type": "application/json; charset=utf-8",
}
token = str(self.model_config.get("controller_model_proxy_token") or "")
if token:
headers["Authorization"] = f"Bearer {token}"
started = time.monotonic()
request = Request(url, data=body, headers=headers, method="POST")
try:
with urlopen(request, timeout=self.timeout_seconds) as response:
raw = response.read()
except HTTPError as exc:
error_body = exc.read().decode("utf-8", errors="replace")
return ModelResponse(
ok=False,
duration_seconds=round(time.monotonic() - started, 3),
error=f"controller model proxy -> HTTP {exc.code}: {error_body or exc.reason}",
)
except (URLError, TimeoutError, OSError) as exc:
return ModelResponse(
ok=False,
duration_seconds=round(time.monotonic() - started, 3),
error=_network_error_message("controller model proxy", exc, self.timeout_seconds),
)
duration = round(time.monotonic() - started, 3)
try:
data = json.loads(raw.decode("utf-8")) if raw else {}
except json.JSONDecodeError as exc:
return ModelResponse(ok=False, duration_seconds=duration, error=f"Invalid controller proxy response: {exc}")
content = str(data.get("content") or "")
parsed = data.get("parsed") if isinstance(data.get("parsed"), dict) else parse_json_object(content)
proxy_duration = data.get("duration_seconds")
try:
duration = float(proxy_duration) if proxy_duration is not None else duration
except (TypeError, ValueError):
pass
return ModelResponse(
ok=bool(data.get("ok")),
content=content,
parsed=parsed,
duration_seconds=round(duration, 3),
error=str(data.get("error") or ""),
)
+103
View File
@@ -0,0 +1,103 @@
from __future__ import annotations
HOST_AGENT_SYSTEM_PROMPT = """
你是系统测试 Agent 的 HostAgent,负责把一个待测系统拆解成可执行、可追踪、可恢复的测试任务。
职责:
1. 阅读当前任务、任务层级、历史轨迹、页面观察结果和可用工具。
2. 判断当前任务是否需要继续拆分为子任务。
3. 将当前可执行范围交给 AppAgent,并说明目标、边界和风险。
4. 遇到账号、密码、验证码、权限、网络异常或破坏性操作时,不要硬做,标记为 PENDING 或 CONFIRM,并说明需要用户补充的信息。
5. 所有任务都必须保留层级关系,便于最终汇总成测试报告。
P0 阶段约束:
- 允许打开网页、观察页面、读取标题/状态、识别链接/表单/按钮、保存截图、生成子任务。
- 不允许自动提交真实表单、删除、支付、保存、发送消息或修改业务数据。
- 如果必须继续执行高风险动作,返回 CONFIRM。
输出必须是 JSON 对象:
{
"observation": "当前任务状态",
"thought": "调度判断",
"current_subtask": "分配给 AppAgent 的子任务",
"message": ["给 AppAgent 的约束或提示"],
"status": "FINISH | CONTINUE | PENDING | ASSIGN | FAIL",
"plan": ["后续子任务或测试方向"],
"function": "assign_app_agent | finish | ask_user | fail",
"arguments": {},
"questions": [],
"result": "调度结论"
}
""".strip()
APP_AGENT_SYSTEM_PROMPT = """
你是系统测试 Agent 的 AppAgent,负责在单个网页或应用范围内执行 HostAgent 分配的测试子任务。
职责:
1. 打开待测页面并观察当前状态。
2. 识别页面中的功能入口:导航链接、按钮、表单、搜索框、登录入口、上传入口等。
3. 记录测试证据:URL、标题、HTTP 状态、可交互元素、截图路径、失败原因。
4. 如果页面需要账号密码、验证码、权限或网络不可达,标记 PENDING 并说明缺少什么。
P0 阶段约束:
- 只做安全观察和无副作用动作。
- 可以打开页面、读取结构、保存截图、枚举窗口和控件。
- 不要自动提交真实表单、删除、支付、保存、发送消息或修改业务数据。
- 高风险入口只记录并生成待确认任务,不要点击执行。
输出必须是 JSON 对象,不要输出 Markdown:
{
"observation": "当前页面或应用状态",
"thought": "为什么选择这一步",
"action": {
"function": "open_in_browser | observe_web_page | discover_web_subtasks | list_windows | inspect_window_controls | capture_desktop | capture_window | no_action",
"arguments": {},
"status": "CONTINUE | FINISH | FAIL | CONFIRM | PENDING"
},
"plan": ["后续动作或子任务方向"],
"comment": "给用户或上层 Agent 的简短说明",
"save_screenshot": {"save": false, "reason": ""},
"result": "本轮执行结论"
}
""".strip()
SYSTEM_TEST_ANALYST_PROMPT = """
你是系统功能测试分析 Agent。你会收到一个页面观察结果、当前任务层级、可用链接/表单/按钮和历史记忆。
你的目标不是复述 HTML,而是像测试工程师一样判断“这个页面上有哪些业务功能值得测试”,并把它们拆成层级化测试子任务。
要求:
1. 根据页面标题、URL、链接文本、按钮文本、表单字段等推断功能模块。
2. 优先生成用户可理解的功能测试任务,例如“登录表单校验”“系统测试启动流程”“模型配置保存”“任务详情查看”。
3. 每个子任务必须给出测试目标 objective,而不是只给元素名称。
4. 如果需要账号、密码、验证码、权限或测试数据,写入 preconditions。
5. 识别风险:low 表示只读/导航/查看;medium 表示保存/提交/上传/新增/修改;high 表示删除/支付/清空/注销/不可逆。
6. P0 阶段不要求执行破坏性操作,只生成待确认任务。
7. 不要生成重复任务;最多生成 8 个高价值子任务。
8. 如果当前页面信息不足,也要说明原因,并可以返回空数组。
输出必须是严格 JSON 对象,不要输出 Markdown:
{
"summary": "对当前页面业务能力的简短判断",
"coverage_notes": ["本轮能够覆盖或不能覆盖的点"],
"needs_user_input": false,
"user_input_reason": "",
"functional_tasks": [
{
"title": "用户可读的测试任务标题",
"objective": "具体测试目标和通过标准",
"module": "所属模块",
"target_url": "可访问 URL,无法判断则留空",
"level": "module | feature | case",
"risk_level": "low | medium | high",
"priority": 10,
"preconditions": [],
"source": "llm_analysis",
"reason": "为什么生成这个任务"
}
]
}
""".strip()
+848
View File
@@ -0,0 +1,848 @@
from __future__ import annotations
import hashlib
import json
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable
from urllib.parse import urljoin, urlparse
from .memory import Blackboard
from .model_client import ModelClient
from .prompts import APP_AGENT_SYSTEM_PROMPT, HOST_AGENT_SYSTEM_PROMPT, SYSTEM_TEST_ANALYST_PROMPT
from .tools import AgentToolRegistry, ToolResult, normalize_url
SubmitTask = Callable[[dict[str, Any]], dict[str, Any]]
@dataclass
class AgentSessionResult:
payload: dict[str, Any]
class AgentSession:
"""
Minimal UFO-inspired execution loop for one claimed task.
The implementation keeps the same conceptual boundaries as UFO:
HostAgent decides the subtask, AppAgent performs one safe step, Blackboard
persists requests/observations/trajectories for later rounds.
"""
def __init__(
self,
*,
task: dict[str, Any],
model_config_state: dict[str, Any],
memory_path: Path,
request_timeout_seconds: float,
submit_task: SubmitTask | None = None,
) -> None:
self.task = task
self.model_config_state = model_config_state
self.model_config = model_config_state.get("value") if isinstance(model_config_state.get("value"), dict) else {}
self.request_timeout_seconds = request_timeout_seconds
self.submit_task = submit_task
self.blackboard = Blackboard(memory_path)
self.tools = AgentToolRegistry()
self.model = ModelClient(self.model_config, timeout_seconds=self._model_timeout_seconds())
self.logs: list[str] = []
self.screenshot_dir = memory_path.parent / "screenshots"
def _model_timeout_seconds(self) -> float:
try:
timeout = float(self.model_config.get("request_timeout_seconds") or self.request_timeout_seconds)
except (TypeError, ValueError):
timeout = self.request_timeout_seconds
return max(1.0, min(timeout, 300.0))
@property
def payload(self) -> dict[str, Any]:
value = self.task.get("payload")
return value if isinstance(value, dict) else {}
@property
def task_id(self) -> str:
return str(self.task.get("external_id") or self.task.get("id") or "")
def target_url(self) -> str:
return normalize_url(
self.payload.get("target_url")
or self.payload.get("url")
or self.payload.get("site_url")
or self.task.get("target_url")
or self.model_config.get("target_system_url")
)
def run(self) -> dict[str, Any]:
started = time.monotonic()
self.blackboard.append(
"requests",
{
"task": self._task_brief(),
"model_config_version": self.model_config_state.get("version"),
},
task_id=self.task_id,
)
host_decision = self._host_decision()
if self.execution_mode() == "gui":
return self._run_gui_task(started, host_decision)
target_url = self.target_url()
if not target_url:
return self._finish_without_target(started, host_decision)
open_browser_result: ToolResult | None = None
if self.payload.get("open_in_browser", True):
open_browser_result = self.tools.web.open_in_browser(
target_url,
wait_seconds=float(self.payload.get("browser_wait_seconds") or 1.5),
)
self.blackboard.append(
"actions",
{
"tool": "open_in_browser",
"ok": open_browser_result.ok,
"error": open_browser_result.error,
"result": open_browser_result.result,
},
task_id=self.task_id,
)
if open_browser_result.ok:
self.logs.append(f"Opened target URL in browser: {target_url}")
else:
self.logs.append(f"Browser open failed: {open_browser_result.error}")
observation_result = self.tools.web.observe_page(
target_url,
timeout_seconds=self.request_timeout_seconds,
)
observation = observation_result.result
desktop_screenshot_result: ToolResult | None = None
if self.payload.get("capture_screenshot", True):
desktop_screenshot_result = self.tools.gui.capture_desktop(
self.screenshot_dir,
prefix=f"task_{self.task.get('id') or self.task_id}_web",
)
if desktop_screenshot_result.ok:
self.blackboard.append(
"screenshots",
{
"image_path": desktop_screenshot_result.result.get("image_path"),
"scope": "web_desktop",
"target_url": target_url,
},
task_id=self.task_id,
)
elif desktop_screenshot_result.error:
self.logs.append(f"Desktop screenshot failed: {desktop_screenshot_result.error}")
self.blackboard.append(
"observations",
{
"tool": "observe_web_page",
"ok": observation_result.ok,
"error": observation_result.error,
"browser_open": open_browser_result.result if open_browser_result else None,
"screenshot": desktop_screenshot_result.result if desktop_screenshot_result and desktop_screenshot_result.ok else None,
"summary": self._observation_summary(observation),
},
task_id=self.task_id,
)
app_decision = self._app_decision(host_decision, observation_result)
discovered_children: list[dict[str, Any]] = []
model_generated_children: list[dict[str, Any]] = []
rule_discovered_children: list[dict[str, Any]] = []
submitted_children: list[dict[str, Any]] = []
model_analysis: dict[str, Any] = {
"model_used": False,
"model_available": self.model.available,
}
discovery_source = "none"
if self._allow_child_discovery(observation_result):
model_analysis, model_generated_children = self._model_discover_child_tasks(observation_result)
if model_generated_children:
discovered_children = model_generated_children
discovery_source = "llm"
else:
rule_discovered_children = self.tools.web.discover_child_tasks(
self.task,
observation,
max_depth=int(self.payload.get("discovery_max_depth") or 2),
)
discovered_children = rule_discovered_children
discovery_source = "rule"
if model_analysis.get("model_available") and model_analysis.get("model_used"):
self.logs.append("Model returned no usable functional tasks; used rule-based fallback.")
submitted_children = self._submit_child_tasks(discovered_children)
success, message, needs_user_input = self._status_from_observation(observation_result, app_decision)
if model_analysis.get("needs_user_input"):
needs_user_input = True
success = False
message = str(model_analysis.get("user_input_reason") or "Model determined that user input is required.")
duration = round(time.monotonic() - started, 3)
result = {
"success": success,
"message": message,
"task_type": self.payload.get("task_type") or "manual",
"agent_framework": "ufo-inspired-lite",
"p0_mode": True,
"needs_user_input": needs_user_input,
"target_url": target_url,
"final_url": observation.get("final_url"),
"http_status": observation.get("http_status"),
"content_type": observation.get("content_type"),
"page_title": observation.get("page_title"),
"duration_seconds": duration,
"action_duration_seconds": round(
observation_result.duration_seconds
+ (open_browser_result.duration_seconds if open_browser_result else 0)
+ (desktop_screenshot_result.duration_seconds if desktop_screenshot_result else 0),
3,
),
"model_duration_seconds": round(
self._model_duration(host_decision, app_decision)
+ float(model_analysis.get("model_duration_seconds") or 0),
3,
),
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
"host_decision": host_decision,
"app_decision": app_decision,
"model_available": self.model.available,
"model_analysis": model_analysis,
"model_errors": self._model_errors(host_decision, app_decision, model_analysis),
"observation": self._observation_summary(observation),
"browser_open": open_browser_result.result if open_browser_result else None,
"screenshots": [
desktop_screenshot_result.result
] if desktop_screenshot_result and desktop_screenshot_result.ok else [],
"available_tools": self.tools.descriptions(),
"discovered_child_tasks": self._child_task_briefs(discovered_children),
"model_generated_child_tasks": self._child_task_briefs(model_generated_children),
"rule_discovered_child_tasks": self._child_task_briefs(rule_discovered_children),
"discovery_source": discovery_source,
"submitted_child_tasks": submitted_children,
"memory": self.blackboard.summary(),
"logs": [*self.logs, *(observation_result.result.get("logs") or [])],
}
self.blackboard.append(
"trajectories",
{
"host_decision": host_decision,
"app_decision": app_decision,
"result": {
"success": success,
"message": message,
"submitted_child_count": len(submitted_children),
"duration_seconds": duration,
},
},
task_id=self.task_id,
)
result["memory"] = self.blackboard.summary()
return result
def execution_mode(self) -> str:
explicit = str(self.payload.get("execution_mode") or self.payload.get("agent_mode") or "").strip().lower()
if explicit in {"web", "gui"}:
return explicit
task_type = str(self.payload.get("task_type") or "").lower()
if task_type.startswith("gui") or self.payload.get("window_id") or self.payload.get("window_title"):
return "gui"
return "web"
def _run_gui_task(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]:
windows_result = self.tools.gui.list_windows()
screenshot_result = self.tools.gui.capture_desktop(self.screenshot_dir, prefix=f"task_{self.task.get('id') or self.task_id}_desktop")
selected_window = self._select_window(windows_result.result.get("windows") or [])
inspect_result: ToolResult | None = None
window_screenshot_result: ToolResult | None = None
if selected_window:
inspect_result = self.tools.gui.inspect_window(str(selected_window["id"]))
window_screenshot_result = self.tools.gui.capture_window(
str(selected_window["id"]),
self.screenshot_dir,
prefix=f"task_{self.task.get('id') or self.task_id}_window",
)
observation = self._gui_observation_summary(
windows_result,
screenshot_result,
selected_window,
inspect_result,
window_screenshot_result,
)
self.blackboard.append(
"observations",
{
"tool": "gui_observe",
"ok": windows_result.ok,
"error": windows_result.error,
"summary": observation,
},
task_id=self.task_id,
)
if screenshot_result.ok:
self.blackboard.append(
"screenshots",
{"image_path": screenshot_result.result.get("image_path"), "scope": "desktop"},
task_id=self.task_id,
)
if window_screenshot_result and window_screenshot_result.ok:
self.blackboard.append(
"screenshots",
{"image_path": window_screenshot_result.result.get("image_path"), "scope": "window", "window": selected_window},
task_id=self.task_id,
)
app_decision = {
"observation": json.dumps(observation, ensure_ascii=False),
"thought": "GUI P0 阶段只枚举窗口、控件并保存证据截图,不执行点击或输入。",
"action": {
"function": "list_windows" if not selected_window else "inspect_window_controls",
"arguments": {
"window_id": selected_window.get("id") if selected_window else None,
"window_title": self.payload.get("window_title"),
},
"status": "FINISH" if windows_result.ok and (not self.payload.get("window_title") or selected_window) else "FAIL",
},
"plan": [],
"comment": "GUI 观察完成。" if windows_result.ok else windows_result.error,
"save_screenshot": {
"save": bool(screenshot_result.ok or (window_screenshot_result and window_screenshot_result.ok)),
"reason": "GUI P0 观察需要截图作为证据。" if screenshot_result.ok else screenshot_result.error,
},
"result": "GUI 窗口和控件信息已记录。" if windows_result.ok else windows_result.error,
"model_used": False,
}
success = windows_result.ok and (not self.payload.get("window_title") or bool(selected_window))
message = "Agent GUI P0 observation completed." if success else "Requested GUI window was not found or could not be inspected."
duration = round(time.monotonic() - started, 3)
result = {
"success": success,
"message": message,
"task_type": self.payload.get("task_type") or "gui_p0",
"agent_framework": "ufo-inspired-lite",
"p0_mode": True,
"execution_mode": "gui",
"needs_user_input": False,
"duration_seconds": duration,
"action_duration_seconds": round(
windows_result.duration_seconds
+ screenshot_result.duration_seconds
+ (inspect_result.duration_seconds if inspect_result else 0)
+ (window_screenshot_result.duration_seconds if window_screenshot_result else 0),
3,
),
"model_duration_seconds": self._model_duration(host_decision, app_decision),
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
"host_decision": host_decision,
"app_decision": app_decision,
"observation": observation,
"available_tools": self.tools.descriptions(),
"memory": self.blackboard.summary(),
"logs": [
*self.logs,
*(windows_result.error and [windows_result.error] or []),
*(screenshot_result.error and [screenshot_result.error] or []),
*(inspect_result and inspect_result.error and [inspect_result.error] or []),
],
}
self.blackboard.append(
"trajectories",
{
"host_decision": host_decision,
"app_decision": app_decision,
"result": {
"success": success,
"message": message,
"duration_seconds": duration,
},
},
task_id=self.task_id,
)
result["memory"] = self.blackboard.summary()
return result
def _host_decision(self) -> dict[str, Any]:
fallback = {
"observation": "收到一个主控端分配的测试任务。",
"thought": "P0 阶段先把任务限定到单个 Web/AppAgent,执行无副作用观察,并按页面结构生成后续子任务。",
"current_subtask": self.payload.get("objective") or self.payload.get("instruction") or self.task.get("title") or "",
"message": ["只执行无副作用观察动作;遇到登录、验证码、权限或破坏性入口时暂停并回传。"],
"status": "ASSIGN",
"plan": ["观察目标页面", "记录页面元信息", "识别可继续拆分的链接、表单和入口"],
"function": "assign_app_agent",
"arguments": {"agent": "web_app_agent", "target_url": self.target_url()},
"questions": [],
"result": "已分配给 Web AppAgent 执行 P0 观察。",
"model_used": False,
}
if not self.model.available:
return fallback
context = {
"task": self._task_brief(),
"blackboard": self.blackboard.snapshot(limit=5),
"tools": self.tools.descriptions(),
}
response = self.model.chat_json(
[
{"role": "system", "content": HOST_AGENT_SYSTEM_PROMPT},
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
]
)
if response.ok and isinstance(response.parsed, dict):
parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds}
return parsed
self.logs.append(f"HostAgent model fallback: {response.error or 'invalid JSON response'}")
return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds}
def _app_decision(self, host_decision: dict[str, Any], observation_result: ToolResult) -> dict[str, Any]:
observation = self._observation_summary(observation_result.result)
fallback_status = "FINISH" if observation_result.ok else "FAIL"
if observation.get("requires_auth"):
fallback_status = "PENDING"
fallback = {
"observation": json.dumps(observation, ensure_ascii=False),
"thought": "P0 阶段采用 observe_web_page 工具完成可达性和页面结构观察,不执行提交、保存、删除等有副作用动作。",
"action": {
"function": "observe_web_page",
"arguments": {"target_url": self.target_url()},
"status": fallback_status,
},
"plan": [],
"comment": "页面观察完成。" if observation_result.ok else observation_result.error,
"save_screenshot": {"save": False, "reason": "P0 Web 观察暂不保存截图。"},
"result": "页面可达性和结构信息已记录。" if observation_result.ok else observation_result.error,
"model_used": False,
}
if not self.model.available:
return fallback
context = {
"host_decision": host_decision,
"task": self._task_brief(),
"observation": observation,
"available_tools": self.tools.descriptions(),
"recent_trajectories": self.blackboard.recent("trajectories", 5),
}
response = self.model.chat_json(
[
{"role": "system", "content": APP_AGENT_SYSTEM_PROMPT},
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
]
)
if response.ok and isinstance(response.parsed, dict):
parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds}
return parsed
self.logs.append(f"AppAgent model fallback: {response.error or 'invalid JSON response'}")
return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds}
def _model_discover_child_tasks(self, observation_result: ToolResult) -> tuple[dict[str, Any], list[dict[str, Any]]]:
base_analysis: dict[str, Any] = {
"model_available": self.model.available,
"model_called": False,
"model_call_succeeded": False,
"model_used": False,
"model_duration_seconds": 0,
"summary": "",
"coverage_notes": [],
"needs_user_input": False,
"user_input_reason": "",
"functional_task_count": 0,
}
if not observation_result.ok:
return {**base_analysis, "error": observation_result.error or "Page observation failed."}, []
if not self.model.available:
self.logs.append("Model config is incomplete; used rule-based discovery fallback.")
return {**base_analysis, "error": "Model config is incomplete."}, []
observation = observation_result.result
context = {
"task": self._task_brief(),
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
"page": self._observation_for_model(observation),
"recent_memory": self.blackboard.snapshot(limit=4),
"constraints": {
"p0_no_side_effects": True,
"max_functional_tasks": 8,
"discovery_max_depth": int(self.payload.get("discovery_max_depth") or 2),
},
}
response = self.model.chat_json(
[
{"role": "system", "content": SYSTEM_TEST_ANALYST_PROMPT},
{"role": "user", "content": json.dumps(context, ensure_ascii=False)},
]
)
if not response.ok or not isinstance(response.parsed, dict):
self.logs.append(f"SystemTestAnalyst model fallback: {response.error or 'invalid JSON response'}")
return {
**base_analysis,
"model_called": True,
"model_call_succeeded": False,
"model_used": False,
"model_duration_seconds": response.duration_seconds,
"error": response.error or "Model did not return a valid JSON object.",
}, []
parsed = response.parsed
children = self._children_from_model_analysis(parsed, observation)
analysis = {
**base_analysis,
"model_called": True,
"model_call_succeeded": True,
"model_used": True,
"model_duration_seconds": response.duration_seconds,
"summary": str(parsed.get("summary") or ""),
"coverage_notes": parsed.get("coverage_notes") if isinstance(parsed.get("coverage_notes"), list) else [],
"needs_user_input": bool(parsed.get("needs_user_input")),
"user_input_reason": str(parsed.get("user_input_reason") or ""),
"functional_task_count": len(children),
}
self.logs.append(
"Model analyzed page and generated "
f"{len(children)} functional task(s): {analysis.get('summary') or 'no summary'}"
)
return analysis, children
def _observation_for_model(self, observation: dict[str, Any]) -> dict[str, Any]:
def clean_items(items: Any, limit: int) -> list[dict[str, Any]]:
if not isinstance(items, list):
return []
cleaned: list[dict[str, Any]] = []
for item in items[:limit]:
if not isinstance(item, dict):
continue
cleaned.append(
{
key: value
for key, value in item.items()
if key in {"href", "text", "action", "method", "id", "name", "class", "type", "fields", "placeholder", "aria_label", "in_form"}
}
)
return cleaned
return {
"url": observation.get("url"),
"final_url": observation.get("final_url"),
"http_status": observation.get("http_status"),
"content_type": observation.get("content_type"),
"page_title": observation.get("page_title"),
"requires_auth": observation.get("requires_auth"),
"has_login_form": observation.get("has_login_form"),
"links": clean_items(observation.get("links"), 20),
"forms": clean_items(observation.get("forms"), 12),
"buttons": clean_items(observation.get("buttons"), 20),
}
def _children_from_model_analysis(self, analysis: dict[str, Any], observation: dict[str, Any]) -> list[dict[str, Any]]:
raw_items = analysis.get("functional_tasks")
if not isinstance(raw_items, list):
return []
max_children = 8
children: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
for index, item in enumerate(raw_items[:max_children], start=1):
if not isinstance(item, dict):
continue
child = self._child_from_model_item(item, observation, index)
key = (
str(child.get("title") or "").strip().lower(),
str((child.get("payload") or {}).get("target_url") or "").strip().lower(),
)
if key in seen:
continue
seen.add(key)
children.append(child)
return children
def _child_from_model_item(self, item: dict[str, Any], observation: dict[str, Any], index: int) -> dict[str, Any]:
parent_external_id = str(self.task.get("external_id") or self.task.get("id") or "")
root_external_id = str(self.task.get("root_external_id") or self.payload.get("root_external_id") or parent_external_id)
parent_depth = int(self.task.get("depth") or self.payload.get("depth") or 0)
depth = parent_depth + 1
discovery_max_depth = int(self.payload.get("discovery_max_depth") or 2)
base_url = observation.get("final_url") or observation.get("url") or self.target_url()
raw_target_url = str(item.get("target_url") or "").strip()
target_url = urljoin(base_url, raw_target_url) if raw_target_url else base_url
title = str(item.get("title") or f"模型生成测试任务 {index}").strip()
objective = str(item.get("objective") or title).strip()
module = str(item.get("module") or title).strip()
level = str(item.get("level") or "feature").strip().lower()
if level not in {"module", "feature", "case"}:
level = "feature"
risk = str(item.get("risk_level") or "low").strip().lower()
if risk not in {"low", "medium", "high"}:
risk = "low"
try:
priority = int(item.get("priority") or 30 + depth * 10 + index)
except (TypeError, ValueError):
priority = 30 + depth * 10 + index
preconditions = item.get("preconditions") if isinstance(item.get("preconditions"), list) else []
external_id = self._stable_child_id(root_external_id, parent_external_id, title, target_url, objective)
payload = {
"task_type": "smart_discovery_p0",
"instruction": objective,
"objective": objective,
"target_url": target_url,
"source": "llm_analysis",
"llm_reason": item.get("reason") or "",
"level": level,
"depth": depth,
"risk_level": risk,
"allow_child_discovery": level in {"module", "feature"} and depth < discovery_max_depth,
"discovery_max_depth": discovery_max_depth,
"open_in_browser": True,
"capture_screenshot": True,
"preconditions": preconditions,
"hierarchy": {
"id": external_id,
"parent_id": parent_external_id,
"root_id": root_external_id,
"depth": depth,
"level": level,
},
}
return {
"external_id": external_id,
"title": title,
"module": module,
"parent_external_id": parent_external_id,
"root_external_id": root_external_id,
"depth": depth,
"task_level": level,
"risk_level": risk,
"priority": priority,
"failure_policy": "pause" if risk in {"medium", "high"} or preconditions else "skip",
"max_retries": 2,
"timeout_seconds": 90,
"tags": ["llm-generated", "system-test", "p0"],
"payload": payload,
}
def _stable_child_id(self, *parts: Any) -> str:
raw = "|".join(str(part or "") for part in parts)
return f"llm-{hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]}"
def _allow_child_discovery(self, observation_result: ToolResult) -> bool:
if not observation_result.ok:
return False
allow = self.payload.get("allow_child_discovery")
if allow is not None:
return bool(allow)
if self.payload.get("task_type") != "smart_discovery_p0":
return False
level = str(self.payload.get("level") or self.task.get("task_level") or "")
depth = int(self.payload.get("depth") or self.task.get("depth") or 0)
return level in {"site", "module", ""} and depth < int(self.payload.get("discovery_max_depth") or 2)
def _submit_child_tasks(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not children or self.submit_task is None:
return []
submitted: list[dict[str, Any]] = []
plan_name = self.task.get("plan_name") or self.payload.get("plan_name") or "智能探索 P0"
for child in children:
payload = {**child, "plan_name": plan_name}
try:
created = self.submit_task(payload)
except Exception as exc:
self.logs.append(f"Submit child task failed: {child.get('external_id')} error={exc}")
continue
submitted.append(
{
"external_id": created.get("external_id") or child.get("external_id"),
"title": created.get("title") or child.get("title"),
"depth": created.get("depth") if created.get("depth") is not None else child.get("depth"),
"task_level": created.get("task_level") or child.get("task_level"),
}
)
return submitted
def _status_from_observation(self, observation_result: ToolResult, app_decision: dict[str, Any]) -> tuple[bool, str, bool]:
action = app_decision.get("action") if isinstance(app_decision.get("action"), dict) else {}
status = str(action.get("status") or "").upper()
if status == "PENDING":
return False, "Task needs user input before it can continue.", True
if status in {"FAIL", "CONFIRM"}:
return False, app_decision.get("result") or app_decision.get("comment") or "Agent action did not complete.", status == "CONFIRM"
if not observation_result.ok:
return False, observation_result.error or "Target observation failed.", False
if observation_result.result.get("http_status") in {401, 403}:
return False, "Target page requires authentication or permission.", True
return True, "Agent P0 observation completed.", False
def _finish_without_target(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]:
task_type = self.payload.get("task_type") or "manual"
success = task_type != "smart_discovery_p0"
duration = round(time.monotonic() - started, 3)
result = {
"success": success,
"message": "No executable target URL was provided." if not success else "Task accepted; no executable URL was provided for P0 mode.",
"task_type": task_type,
"agent_framework": "ufo-inspired-lite",
"p0_mode": True,
"needs_user_input": not success,
"duration_seconds": duration,
"hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(),
"host_decision": host_decision,
"app_decision": {
"action": {"function": "no_action", "arguments": {}, "status": "FAIL" if not success else "FINISH"},
"result": "Missing target URL.",
},
"available_tools": self.tools.descriptions(),
"memory": self.blackboard.summary(),
"logs": ["AgentSession requires target_url for Web P0 execution."],
}
self.blackboard.append("trajectories", {"host_decision": host_decision, "result": result}, task_id=self.task_id)
return result
def _select_window(self, windows: list[dict[str, Any]]) -> dict[str, Any] | None:
window_id = self.payload.get("window_id")
if window_id:
for window in windows:
if str(window.get("id")) == str(window_id) or str(window.get("hex_id")).lower() == str(window_id).lower():
return window
return None
title = str(self.payload.get("window_title") or self.payload.get("application") or "").strip().lower()
if not title:
return None
for window in windows:
if title in str(window.get("title") or "").lower():
return window
for window in windows:
if title in str(window.get("class_name") or "").lower():
return window
return None
def _gui_observation_summary(
self,
windows_result: ToolResult,
screenshot_result: ToolResult,
selected_window: dict[str, Any] | None,
inspect_result: ToolResult | None,
window_screenshot_result: ToolResult | None,
) -> dict[str, Any]:
windows = windows_result.result.get("windows") if isinstance(windows_result.result, dict) else []
controls = []
if inspect_result and isinstance(inspect_result.result, dict):
controls = inspect_result.result.get("controls") or []
return {
"backend": (windows_result.result or {}).get("backend") or self.tools.gui.backend_status(),
"window_count": len(windows or []),
"windows": [
{
"id": window.get("id"),
"title": window.get("title"),
"class_name": window.get("class_name"),
"process_id": window.get("process_id"),
"rect": window.get("rect"),
"minimized": window.get("minimized"),
}
for window in (windows or [])[:20]
],
"selected_window": selected_window,
"control_count": len(controls),
"controls": [
{
"id": control.get("id"),
"text": control.get("text"),
"class_name": control.get("class_name"),
"rect": control.get("rect"),
"visible": control.get("visible"),
"enabled": control.get("enabled"),
}
for control in controls[:80]
],
"desktop_screenshot": screenshot_result.result if screenshot_result.ok else None,
"window_screenshot": window_screenshot_result.result if window_screenshot_result and window_screenshot_result.ok else None,
"errors": [
message
for message in [
windows_result.error,
screenshot_result.error,
inspect_result.error if inspect_result else "",
window_screenshot_result.error if window_screenshot_result else "",
]
if message
],
}
def _task_brief(self) -> dict[str, Any]:
return {
"id": self.task.get("id"),
"external_id": self.task.get("external_id"),
"title": self.task.get("title"),
"plan_name": self.task.get("plan_name"),
"module": self.task.get("module"),
"status": self.task.get("status"),
"depth": self.task.get("depth"),
"task_level": self.task.get("task_level"),
"risk_level": self.task.get("risk_level"),
"payload": self.payload,
}
def _task_hierarchy(self) -> dict[str, Any]:
return {
"id": self.task.get("external_id") or self.task.get("id"),
"parent_id": self.task.get("parent_external_id"),
"root_id": self.task.get("root_external_id") or self.task.get("external_id") or self.task.get("id"),
"depth": self.task.get("depth") or 0,
"level": self.task.get("task_level") or self.payload.get("level") or "",
}
def _observation_summary(self, observation: dict[str, Any]) -> dict[str, Any]:
return {
"url": observation.get("url"),
"final_url": observation.get("final_url"),
"http_status": observation.get("http_status"),
"content_type": observation.get("content_type"),
"page_title": observation.get("page_title"),
"requires_auth": observation.get("requires_auth"),
"auth_challenge": observation.get("auth_challenge"),
"has_login_form": observation.get("has_login_form"),
"link_count": len(observation.get("links") or []),
"form_count": len(observation.get("forms") or []),
"button_count": len(observation.get("buttons") or []),
}
def _child_task_briefs(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [
{
"external_id": child.get("external_id"),
"title": child.get("title"),
"parent_external_id": child.get("parent_external_id"),
"root_external_id": child.get("root_external_id"),
"depth": child.get("depth"),
"task_level": child.get("task_level"),
"risk_level": child.get("risk_level"),
"target_url": (child.get("payload") or {}).get("target_url"),
"source": (child.get("payload") or {}).get("source"),
"preconditions": (child.get("payload") or {}).get("preconditions") or [],
"llm_reason": (child.get("payload") or {}).get("llm_reason") or "",
}
for child in children
]
def _model_duration(self, host_decision: dict[str, Any], app_decision: dict[str, Any]) -> float:
return round(
float(host_decision.get("model_duration_seconds") or 0)
+ float(app_decision.get("model_duration_seconds") or 0),
3,
)
def _model_errors(self, *parts: dict[str, Any]) -> list[str]:
errors: list[str] = []
for part in parts:
error = str(part.get("model_error") or part.get("error") or "").strip()
if error and error not in errors:
errors.append(error)
return errors
+790
View File
@@ -0,0 +1,790 @@
from __future__ import annotations
import hashlib
import html
import platform
import re
import ssl
import time
import webbrowser
import ctypes
import ctypes.wintypes
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urldefrag, urljoin, urlparse
from urllib.request import Request, urlopen
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
UNVERIFIED_HTTPS_CONTEXT = ssl._create_unverified_context()
MAX_HTML_BYTES = 1_500_000
MAX_CHILD_TASKS = 8
def network_error_message(exc: BaseException, timeout_seconds: float) -> str:
reason = getattr(exc, "reason", exc)
if isinstance(exc, TimeoutError) or isinstance(reason, TimeoutError):
return f"timed out after {timeout_seconds:g}s"
return str(reason) or type(reason).__name__
@dataclass
class ToolResult:
ok: bool
result: dict[str, Any] = field(default_factory=dict)
error: str = ""
duration_seconds: float = 0.0
def clean_text(value: Any, max_length: int = 160) -> str:
text = html.unescape(re.sub(r"\s+", " ", str(value or ""))).strip()
return text[:max_length].strip()
def decode_body(raw: bytes, content_type: str) -> str:
charset = "utf-8"
match = re.search(r"charset=([^;\s]+)", content_type or "", re.IGNORECASE)
if match:
charset = match.group(1).strip("\"'")
for candidate in [charset, "utf-8", "gbk", "latin-1"]:
try:
return raw.decode(candidate, errors="replace")
except LookupError:
continue
return raw.decode("utf-8", errors="replace")
def extract_title(raw: bytes, content_type: str) -> str:
text = decode_body(raw, content_type)
match = TITLE_RE.search(text)
if not match:
return ""
return clean_text(match.group(1), 120)
def normalize_url(value: Any) -> str:
text = str(value or "").strip()
if text and "://" not in text:
text = f"http://{text}"
return text
def origin_tuple(url: str) -> tuple[str, str, int]:
parsed = urlparse(url)
port = parsed.port or (443 if parsed.scheme == "https" else 80)
return parsed.scheme, (parsed.hostname or "").lower(), port
def same_origin(base_url: str, candidate_url: str) -> bool:
try:
return origin_tuple(base_url) == origin_tuple(candidate_url)
except ValueError:
return False
def safe_join_url(base_url: str, href: str) -> str | None:
text = str(href or "").strip()
if not text or text.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
return None
joined = urljoin(base_url, text)
joined, _fragment = urldefrag(joined)
parsed = urlparse(joined)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
return None
if not same_origin(base_url, joined):
return None
return joined
def stable_id(*parts: Any, prefix: str = "agent") -> str:
raw = "|".join(str(part or "") for part in parts)
return f"{prefix}-{hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]}"
def risk_for_text(text: str) -> str:
lowered = text.lower()
high = ["delete", "remove", "drop", "pay", "refund", "logout", "删除", "清空", "支付", "付款", "退款", "注销"]
medium = ["save", "submit", "create", "update", "edit", "upload", "保存", "提交", "新增", "修改", "上传"]
if any(word in lowered for word in high):
return "high"
if any(word in lowered for word in medium):
return "medium"
return "low"
class WebPageParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.title_parts: list[str] = []
self.links: list[dict[str, Any]] = []
self.forms: list[dict[str, Any]] = []
self.buttons: list[dict[str, Any]] = []
self._in_title = False
self._anchors: list[dict[str, Any]] = []
self._form: dict[str, Any] | None = None
self._button: dict[str, Any] | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
tag = tag.lower()
attr = {name.lower(): value or "" for name, value in attrs}
if tag == "title":
self._in_title = True
elif tag == "a":
self._anchors.append({"href": attr.get("href", ""), "text_parts": []})
elif tag == "form":
self._form = {
"action": attr.get("action", ""),
"method": (attr.get("method") or "get").upper(),
"id": attr.get("id", ""),
"name": attr.get("name", ""),
"class": attr.get("class", ""),
"text_parts": [],
"fields": [],
}
elif self._form is not None and tag in {"input", "select", "textarea"}:
self._form["fields"].append(
{
"tag": tag,
"type": (attr.get("type") or tag).lower(),
"name": attr.get("name", ""),
"id": attr.get("id", ""),
"placeholder": attr.get("placeholder", ""),
"aria_label": attr.get("aria-label", ""),
}
)
elif tag == "button":
self._button = {
"type": attr.get("type", ""),
"id": attr.get("id", ""),
"name": attr.get("name", ""),
"class": attr.get("class", ""),
"text_parts": [],
"in_form": self._form is not None,
}
def handle_data(self, data: str) -> None:
if self._in_title:
self.title_parts.append(data)
if self._anchors:
self._anchors[-1]["text_parts"].append(data)
if self._form is not None:
self._form["text_parts"].append(data)
if self._button is not None:
self._button["text_parts"].append(data)
def handle_endtag(self, tag: str) -> None:
tag = tag.lower()
if tag == "title":
self._in_title = False
elif tag == "a" and self._anchors:
anchor = self._anchors.pop()
self.links.append({"href": anchor.get("href", ""), "text": clean_text(" ".join(anchor.get("text_parts") or []), 120)})
elif tag == "button" and self._button is not None:
button = self._button
button["text"] = clean_text(" ".join(button.get("text_parts") or []), 120)
self.buttons.append(button)
self._button = None
elif tag == "form" and self._form is not None:
form = self._form
form["text"] = clean_text(" ".join(form.get("text_parts") or []), 160)
self.forms.append(form)
self._form = None
def looks_like_login_form(form: dict[str, Any]) -> bool:
fields = form.get("fields") or []
text = " ".join(
[
str(form.get("id") or ""),
str(form.get("name") or ""),
str(form.get("class") or ""),
str(form.get("text") or ""),
*[
" ".join(
[
str(field.get("type") or ""),
str(field.get("name") or ""),
str(field.get("id") or ""),
str(field.get("placeholder") or ""),
str(field.get("aria_label") or ""),
]
)
for field in fields
],
]
).lower()
return "password" in text or any(keyword in text for keyword in ["login", "signin", "sign in", "登录", "账号", "密码"])
class WebTools:
def open_in_browser(self, url: str, wait_seconds: float = 1.5) -> ToolResult:
target_url = normalize_url(url)
if not target_url:
return ToolResult(ok=False, error="Missing target URL.")
started = time.monotonic()
try:
opened = bool(webbrowser.open(target_url, new=2, autoraise=True))
if wait_seconds > 0:
time.sleep(min(max(wait_seconds, 0.0), 5.0))
return ToolResult(
ok=opened,
result={"url": target_url, "opened": opened},
error="" if opened else "Default browser did not accept the URL.",
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(
ok=False,
error=str(exc),
duration_seconds=round(time.monotonic() - started, 3),
)
def observe_page(self, url: str, timeout_seconds: float = 8) -> ToolResult:
target_url = normalize_url(url)
if not target_url:
return ToolResult(ok=False, error="Missing target URL.")
request = Request(
target_url,
headers={
"Accept": "text/html,application/xhtml+xml,*/*",
"User-Agent": "SystemTest-Agent/0.1 WebTools",
},
)
started = time.monotonic()
try:
with urlopen(request, timeout=timeout_seconds, context=UNVERIFIED_HTTPS_CONTEXT) as response:
raw = response.read(MAX_HTML_BYTES)
status = response.status
final_url = response.geturl()
content_type = response.headers.get("Content-Type", "")
except HTTPError as exc:
raw = exc.read(MAX_HTML_BYTES)
status = exc.code
final_url = exc.geturl()
content_type = exc.headers.get("Content-Type", "")
except (URLError, TimeoutError, OSError) as exc:
return ToolResult(
ok=False,
error=f"Target URL is not reachable: {network_error_message(exc, timeout_seconds)}",
duration_seconds=round(time.monotonic() - started, 3),
)
text = decode_body(raw, content_type)
parser = WebPageParser()
try:
parser.feed(text)
except Exception:
pass
title = clean_text(" ".join(parser.title_parts), 120) or extract_title(raw, content_type)
duration = round(time.monotonic() - started, 3)
has_login_form = any(looks_like_login_form(form) for form in parser.forms)
auth_challenge = status in {401, 403}
return ToolResult(
ok=status < 500,
duration_seconds=duration,
result={
"url": target_url,
"final_url": final_url,
"http_status": status,
"content_type": content_type,
"page_title": title,
"elapsed_seconds": duration,
"links": parser.links[:50],
"forms": parser.forms[:20],
"buttons": parser.buttons[:50],
"requires_auth": auth_challenge,
"auth_challenge": auth_challenge,
"has_login_form": has_login_form,
},
)
def discover_child_tasks(
self,
parent_task: dict[str, Any],
observation: dict[str, Any],
*,
max_children: int = MAX_CHILD_TASKS,
max_depth: int = 2,
) -> list[dict[str, Any]]:
payload = parent_task.get("payload") if isinstance(parent_task.get("payload"), dict) else {}
parent_external_id = str(parent_task.get("external_id") or parent_task.get("id") or "")
root_external_id = str(parent_task.get("root_external_id") or payload.get("root_external_id") or parent_external_id)
parent_depth = int(parent_task.get("depth") or payload.get("depth") or 0)
discovery_max_depth = int(payload.get("discovery_max_depth") or max_depth)
child_depth = parent_depth + 1
if child_depth > discovery_max_depth:
return []
page_url = observation.get("final_url") or observation.get("url") or payload.get("target_url") or ""
children: list[dict[str, Any]] = []
seen_urls: set[str] = set()
for link in observation.get("links") or []:
if len(children) >= max_children:
break
href = safe_join_url(page_url, str(link.get("href") or ""))
if not href or href in seen_urls:
continue
seen_urls.add(href)
label = clean_text(link.get("text"), 80) or urlparse(href).path.strip("/") or href
risk = risk_for_text(label)
children.append(
self._child_task(
parent_external_id,
root_external_id,
child_depth,
label,
f"探索页面:{label}",
f"访问并检查页面“{label}”是否可达,记录标题、状态码和可交互入口。",
href,
"module",
risk,
"html_link",
discovery_max_depth,
)
)
for index, form in enumerate(observation.get("forms") or [], start=1):
if len(children) >= max_children:
break
action_url = safe_join_url(page_url, str(form.get("action") or "")) or page_url
is_login = looks_like_login_form(form)
label = "登录表单" if is_login else clean_text(form.get("name") or form.get("id") or form.get("text"), 60) or f"表单 {index}"
risk = "medium" if is_login else risk_for_text(label)
child = self._child_task(
parent_external_id,
root_external_id,
child_depth,
label,
f"识别功能:{label}",
f"分析{label}的输入项与提交入口;P0 只确认元素存在,不执行真实提交。",
action_url,
"feature",
risk,
"html_form",
discovery_max_depth,
)
child["payload"]["preconditions"] = ["需要测试账号/密码"] if is_login else []
children.append(child)
for button in observation.get("buttons") or []:
if len(children) >= max_children:
break
label = clean_text(button.get("text") or button.get("name") or button.get("id"), 60)
if not label:
continue
risk = risk_for_text(label)
child = self._child_task(
parent_external_id,
root_external_id,
child_depth,
label,
f"识别入口:{label}",
f"确认入口“{label}”存在;高风险入口只记录,不执行破坏性动作。",
page_url,
"feature",
risk,
"html_button",
discovery_max_depth,
)
if risk == "high":
child["failure_policy"] = "pause"
children.append(child)
unique: dict[str, dict[str, Any]] = {}
for child in children:
unique.setdefault(str(child["external_id"]), child)
return list(unique.values())[:max_children]
def _child_task(
self,
parent_external_id: str,
root_external_id: str,
depth: int,
module: str,
title: str,
objective: str,
target_url: str,
level: str,
risk: str,
source: str,
discovery_max_depth: int,
) -> dict[str, Any]:
external_id = stable_id(root_external_id, parent_external_id, source, target_url, module)
payload = {
"task_type": "smart_discovery_p0",
"instruction": objective,
"objective": objective,
"target_url": target_url,
"source": source,
"level": level,
"depth": depth,
"risk_level": risk,
"allow_child_discovery": level == "module" and depth < discovery_max_depth,
"discovery_max_depth": discovery_max_depth,
"open_in_browser": True,
"capture_screenshot": True,
"hierarchy": {
"id": external_id,
"parent_id": parent_external_id,
"root_id": root_external_id,
"depth": depth,
"level": level,
},
}
return {
"external_id": external_id,
"title": title,
"module": module,
"parent_external_id": parent_external_id,
"root_external_id": root_external_id,
"depth": depth,
"task_level": level,
"risk_level": risk,
"priority": 40 + depth * 10 + (40 if risk == "high" else 20 if risk == "medium" else 0),
"failure_policy": "pause" if risk == "high" else "skip",
"max_retries": 2,
"timeout_seconds": 60,
"tags": ["agent-discovered", "p0"],
"payload": payload,
}
class GuiTools:
def __init__(self) -> None:
self._user32 = ctypes.windll.user32 if self.available() else None
self._enum_proc_type = ctypes.WINFUNCTYPE(ctypes.c_bool, ctypes.wintypes.HWND, ctypes.wintypes.LPARAM) if self.available() else None
if self._user32 and self._enum_proc_type:
self._user32.EnumWindows.argtypes = [ctypes.c_void_p, ctypes.wintypes.LPARAM]
self._user32.EnumWindows.restype = ctypes.c_bool
self._user32.EnumChildWindows.argtypes = [ctypes.wintypes.HWND, ctypes.c_void_p, ctypes.wintypes.LPARAM]
self._user32.EnumChildWindows.restype = ctypes.c_bool
def available(self) -> bool:
return platform.system() == "Windows" and hasattr(ctypes, "windll")
def backend_status(self) -> dict[str, Any]:
if not self.available():
return {
"available": False,
"backend": "ctypes-win32",
"reason": "Windows GUI APIs are unavailable on this platform.",
}
return {
"available": True,
"backend": "ctypes-win32",
"capabilities": [
"list_windows",
"inspect_window_controls",
"capture_desktop",
"capture_window",
"focus_window",
"click_window_point",
],
}
def list_windows(self, limit: int = 80) -> ToolResult:
started = time.monotonic()
if not self.available():
return ToolResult(ok=False, error=self.backend_status()["reason"])
try:
windows = self._enum_windows(limit=limit)
return ToolResult(
ok=True,
result={"windows": windows, "backend": self.backend_status()},
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(ok=False, error=str(exc), duration_seconds=round(time.monotonic() - started, 3))
def inspect_window(self, window_id: str, limit: int = 220) -> ToolResult:
started = time.monotonic()
if not self.available():
return ToolResult(ok=False, error=self.backend_status()["reason"])
try:
hwnd = self._parse_hwnd(window_id)
if not self._user32.IsWindow(hwnd):
return ToolResult(ok=False, error=f"Window not found: {window_id}")
return ToolResult(
ok=True,
result={
"window": self._window_info(hwnd),
"controls": self._enum_child_windows(hwnd, limit=limit),
"backend": self.backend_status(),
},
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(ok=False, error=str(exc), duration_seconds=round(time.monotonic() - started, 3))
def capture_desktop(self, save_dir: Path, prefix: str = "desktop") -> ToolResult:
started = time.monotonic()
try:
from PIL import ImageGrab
save_dir.mkdir(parents=True, exist_ok=True)
image = ImageGrab.grab(all_screens=True)
path = save_dir / f"{prefix}_{int(time.time() * 1000)}.png"
image.save(path)
return ToolResult(
ok=True,
result={"image_path": str(path), "width": image.width, "height": image.height},
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(ok=False, error=str(exc), duration_seconds=round(time.monotonic() - started, 3))
def capture_window(self, window_id: str, save_dir: Path, prefix: str = "window") -> ToolResult:
started = time.monotonic()
if not self.available():
return ToolResult(ok=False, error=self.backend_status()["reason"])
try:
from PIL import ImageGrab
hwnd = self._parse_hwnd(window_id)
if not self._user32.IsWindow(hwnd):
return ToolResult(ok=False, error=f"Window not found: {window_id}")
rect = self._rect(hwnd)
if rect["width"] <= 0 or rect["height"] <= 0:
return ToolResult(ok=False, error=f"Window has invalid bounds: {window_id}")
save_dir.mkdir(parents=True, exist_ok=True)
image = ImageGrab.grab(bbox=(rect["left"], rect["top"], rect["right"], rect["bottom"]))
path = save_dir / f"{prefix}_{window_id}_{int(time.time() * 1000)}.png"
image.save(path)
return ToolResult(
ok=True,
result={
"image_path": str(path),
"window": self._window_info(hwnd),
"width": image.width,
"height": image.height,
},
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(ok=False, error=str(exc), duration_seconds=round(time.monotonic() - started, 3))
def focus_window(self, window_id: str) -> ToolResult:
started = time.monotonic()
if not self.available():
return ToolResult(ok=False, error=self.backend_status()["reason"])
try:
hwnd = self._parse_hwnd(window_id)
if not self._user32.IsWindow(hwnd):
return ToolResult(ok=False, error=f"Window not found: {window_id}")
self._user32.ShowWindow(hwnd, 9) # SW_RESTORE
ok = bool(self._user32.SetForegroundWindow(hwnd))
return ToolResult(
ok=ok,
result={"window": self._window_info(hwnd), "focused": ok},
error="" if ok else "SetForegroundWindow returned false.",
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(ok=False, error=str(exc), duration_seconds=round(time.monotonic() - started, 3))
def click_window_point(self, window_id: str, x: float, y: float, normalized: bool = True) -> ToolResult:
started = time.monotonic()
if not self.available():
return ToolResult(ok=False, error=self.backend_status()["reason"])
try:
hwnd = self._parse_hwnd(window_id)
if not self._user32.IsWindow(hwnd):
return ToolResult(ok=False, error=f"Window not found: {window_id}")
rect = self._rect(hwnd)
if normalized:
target_x = int(rect["left"] + max(0.0, min(1.0, float(x))) * rect["width"])
target_y = int(rect["top"] + max(0.0, min(1.0, float(y))) * rect["height"])
else:
target_x = int(x)
target_y = int(y)
self.focus_window(window_id)
self._user32.SetCursorPos(target_x, target_y)
self._user32.mouse_event(0x0002, 0, 0, 0, 0) # MOUSEEVENTF_LEFTDOWN
self._user32.mouse_event(0x0004, 0, 0, 0, 0) # MOUSEEVENTF_LEFTUP
return ToolResult(
ok=True,
result={"window_id": str(window_id), "x": target_x, "y": target_y, "normalized": normalized},
duration_seconds=round(time.monotonic() - started, 3),
)
except Exception as exc:
return ToolResult(ok=False, error=str(exc), duration_seconds=round(time.monotonic() - started, 3))
def _parse_hwnd(self, window_id: str) -> int:
text = str(window_id).strip()
if text.lower().startswith("0x"):
return int(text, 16)
return int(text)
def _window_text(self, hwnd: int, max_length: int = 512) -> str:
length = min(max(self._user32.GetWindowTextLengthW(hwnd), 0) + 1, max_length)
buffer = ctypes.create_unicode_buffer(length)
self._user32.GetWindowTextW(hwnd, buffer, length)
return clean_text(buffer.value, 200)
def _class_name(self, hwnd: int) -> str:
buffer = ctypes.create_unicode_buffer(256)
self._user32.GetClassNameW(hwnd, buffer, 256)
return clean_text(buffer.value, 120)
def _rect(self, hwnd: int) -> dict[str, int]:
rect = ctypes.wintypes.RECT()
self._user32.GetWindowRect(hwnd, ctypes.byref(rect))
return {
"left": int(rect.left),
"top": int(rect.top),
"right": int(rect.right),
"bottom": int(rect.bottom),
"width": max(0, int(rect.right - rect.left)),
"height": max(0, int(rect.bottom - rect.top)),
}
def _process_id(self, hwnd: int) -> int:
pid = ctypes.wintypes.DWORD()
self._user32.GetWindowThreadProcessId(hwnd, ctypes.byref(pid))
return int(pid.value)
def _window_info(self, hwnd: int) -> dict[str, Any]:
return {
"id": str(int(hwnd)),
"hex_id": hex(int(hwnd)),
"title": self._window_text(hwnd),
"class_name": self._class_name(hwnd),
"process_id": self._process_id(hwnd),
"visible": bool(self._user32.IsWindowVisible(hwnd)),
"enabled": bool(self._user32.IsWindowEnabled(hwnd)),
"minimized": bool(self._user32.IsIconic(hwnd)),
"rect": self._rect(hwnd),
}
def _enum_windows(self, limit: int = 80) -> list[dict[str, Any]]:
windows: list[dict[str, Any]] = []
enum_proc = self._enum_proc_type
@enum_proc
def callback(hwnd: int, _lparam: int) -> bool:
if len(windows) >= limit:
return False
if not self._user32.IsWindowVisible(hwnd):
return True
info = self._window_info(hwnd)
rect = info["rect"]
if rect["width"] < 40 or rect["height"] < 40:
return True
if not info["title"]:
return True
if info["title"] or info["class_name"]:
info["index"] = len(windows) + 1
windows.append(info)
return True
self._user32.EnumWindows(ctypes.cast(callback, ctypes.c_void_p), ctypes.wintypes.LPARAM(0))
return windows
def _enum_child_windows(self, hwnd: int, limit: int = 220) -> list[dict[str, Any]]:
controls: list[dict[str, Any]] = []
enum_proc = self._enum_proc_type
@enum_proc
def callback(child_hwnd: int, _lparam: int) -> bool:
if len(controls) >= limit:
return False
rect = self._rect(child_hwnd)
if rect["width"] <= 0 or rect["height"] <= 0:
return True
text = self._window_text(child_hwnd)
class_name = self._class_name(child_hwnd)
if not text and not class_name:
return True
controls.append(
{
"id": str(int(child_hwnd)),
"hex_id": hex(int(child_hwnd)),
"index": len(controls) + 1,
"text": text,
"class_name": class_name,
"visible": bool(self._user32.IsWindowVisible(child_hwnd)),
"enabled": bool(self._user32.IsWindowEnabled(child_hwnd)),
"rect": rect,
}
)
return True
self._user32.EnumChildWindows(hwnd, ctypes.cast(callback, ctypes.c_void_p), ctypes.wintypes.LPARAM(0))
return controls
class AgentToolRegistry:
def __init__(self) -> None:
self.web = WebTools()
self.gui = GuiTools()
def descriptions(self) -> list[dict[str, Any]]:
gui_status = self.gui.backend_status()
return [
{
"name": "open_in_browser",
"scope": "web",
"available": True,
"description": "Open the target URL in the controlled device default browser so the operator can see the test start.",
},
{
"name": "observe_web_page",
"scope": "web",
"available": True,
"description": "Open a web page, read status/title, and list links/forms/buttons without side effects.",
},
{
"name": "discover_web_subtasks",
"scope": "web",
"available": True,
"description": "Create hierarchy-aware P0 child tasks from observed links/forms/buttons.",
},
{
"name": "list_windows",
"scope": "gui",
"available": gui_status["available"],
"description": "List visible Windows desktop application windows using a dependency-free ctypes Win32 backend.",
},
{
"name": "inspect_window_controls",
"scope": "gui",
"available": gui_status["available"],
"description": "Enumerate Win32 child controls for a selected window.",
},
{
"name": "capture_desktop",
"scope": "gui",
"available": True,
"description": "Capture the desktop screenshot with PIL ImageGrab and save it as local evidence.",
},
{
"name": "capture_window",
"scope": "gui",
"available": gui_status["available"],
"description": "Capture a selected window region and save it as local evidence.",
},
{
"name": "focus_window",
"scope": "gui",
"available": gui_status["available"],
"description": "Bring a window to foreground. This is an action tool and should be used only when the task allows it.",
},
{
"name": "click_window_point",
"scope": "gui",
"available": gui_status["available"],
"description": "Click a normalized or absolute point in a selected window. This is guarded and not used by default in P0 observation.",
},
{
"name": "no_action",
"scope": "agent",
"available": True,
"description": "Finish, fail, or wait for user input without executing a UI action.",
},
]