from __future__ import annotations import hashlib import json import time from dataclasses import dataclass from pathlib import Path from typing import Any, Callable from urllib.parse import urljoin, urlparse from .memory import Blackboard from .model_client import ModelClient from .prompts import APP_AGENT_SYSTEM_PROMPT, HOST_AGENT_SYSTEM_PROMPT, SYSTEM_TEST_ANALYST_PROMPT from .tools import AgentToolRegistry, ToolResult, normalize_url SubmitTask = Callable[[dict[str, Any]], dict[str, Any]] @dataclass class AgentSessionResult: payload: dict[str, Any] class AgentSession: """ Minimal UFO-inspired execution loop for one claimed task. The implementation keeps the same conceptual boundaries as UFO: HostAgent decides the subtask, AppAgent performs one safe step, Blackboard persists requests/observations/trajectories for later rounds. """ def __init__( self, *, task: dict[str, Any], model_config_state: dict[str, Any], memory_path: Path, request_timeout_seconds: float, submit_task: SubmitTask | None = None, ) -> None: self.task = task self.model_config_state = model_config_state self.model_config = model_config_state.get("value") if isinstance(model_config_state.get("value"), dict) else {} self.request_timeout_seconds = request_timeout_seconds self.submit_task = submit_task self.blackboard = Blackboard(memory_path) self.tools = AgentToolRegistry() self.model = ModelClient(self.model_config, timeout_seconds=self._model_timeout_seconds()) self.logs: list[str] = [] self.screenshot_dir = memory_path.parent / "screenshots" def _model_timeout_seconds(self) -> float: try: timeout = float(self.model_config.get("request_timeout_seconds") or self.request_timeout_seconds) except (TypeError, ValueError): timeout = self.request_timeout_seconds return max(1.0, min(timeout, 300.0)) @property def payload(self) -> dict[str, Any]: value = self.task.get("payload") return value if isinstance(value, dict) else {} @property def task_id(self) -> str: return str(self.task.get("external_id") or self.task.get("id") or "") def target_url(self) -> str: return normalize_url( self.payload.get("target_url") or self.payload.get("url") or self.payload.get("site_url") or self.task.get("target_url") or self.model_config.get("target_system_url") ) def run(self) -> dict[str, Any]: started = time.monotonic() self.blackboard.append( "requests", { "task": self._task_brief(), "model_config_version": self.model_config_state.get("version"), }, task_id=self.task_id, ) host_decision = self._host_decision() if self.execution_mode() == "gui": return self._run_gui_task(started, host_decision) target_url = self.target_url() if not target_url: return self._finish_without_target(started, host_decision) open_browser_result: ToolResult | None = None if self.payload.get("open_in_browser", True): open_browser_result = self.tools.web.open_in_browser( target_url, wait_seconds=float(self.payload.get("browser_wait_seconds") or 1.5), ) self.blackboard.append( "actions", { "tool": "open_in_browser", "ok": open_browser_result.ok, "error": open_browser_result.error, "result": open_browser_result.result, }, task_id=self.task_id, ) if open_browser_result.ok: self.logs.append(f"Opened target URL in browser: {target_url}") else: self.logs.append(f"Browser open failed: {open_browser_result.error}") observation_result = self.tools.web.observe_page( target_url, timeout_seconds=self.request_timeout_seconds, ) observation = observation_result.result desktop_screenshot_result: ToolResult | None = None if self.payload.get("capture_screenshot", True): desktop_screenshot_result = self.tools.gui.capture_desktop( self.screenshot_dir, prefix=f"task_{self.task.get('id') or self.task_id}_web", ) if desktop_screenshot_result.ok: self.blackboard.append( "screenshots", { "image_path": desktop_screenshot_result.result.get("image_path"), "scope": "web_desktop", "target_url": target_url, }, task_id=self.task_id, ) elif desktop_screenshot_result.error: self.logs.append(f"Desktop screenshot failed: {desktop_screenshot_result.error}") self.blackboard.append( "observations", { "tool": "observe_web_page", "ok": observation_result.ok, "error": observation_result.error, "browser_open": open_browser_result.result if open_browser_result else None, "screenshot": desktop_screenshot_result.result if desktop_screenshot_result and desktop_screenshot_result.ok else None, "summary": self._observation_summary(observation), }, task_id=self.task_id, ) app_decision = self._app_decision(host_decision, observation_result) discovered_children: list[dict[str, Any]] = [] model_generated_children: list[dict[str, Any]] = [] rule_discovered_children: list[dict[str, Any]] = [] submitted_children: list[dict[str, Any]] = [] model_analysis: dict[str, Any] = { "model_used": False, "model_available": self.model.available, } discovery_source = "none" if self._allow_child_discovery(observation_result): model_analysis, model_generated_children = self._model_discover_child_tasks(observation_result) if model_generated_children: discovered_children = model_generated_children discovery_source = "llm" else: rule_discovered_children = self.tools.web.discover_child_tasks( self.task, observation, max_depth=int(self.payload.get("discovery_max_depth") or 2), ) discovered_children = rule_discovered_children discovery_source = "rule" if model_analysis.get("model_available") and model_analysis.get("model_used"): self.logs.append("Model returned no usable functional tasks; used rule-based fallback.") submitted_children = self._submit_child_tasks(discovered_children) success, message, needs_user_input = self._status_from_observation(observation_result, app_decision) if model_analysis.get("needs_user_input"): needs_user_input = True success = False message = str(model_analysis.get("user_input_reason") or "Model determined that user input is required.") duration = round(time.monotonic() - started, 3) result = { "success": success, "message": message, "task_type": self.payload.get("task_type") or "manual", "agent_framework": "ufo-inspired-lite", "p0_mode": True, "needs_user_input": needs_user_input, "target_url": target_url, "final_url": observation.get("final_url"), "http_status": observation.get("http_status"), "content_type": observation.get("content_type"), "page_title": observation.get("page_title"), "duration_seconds": duration, "action_duration_seconds": round( observation_result.duration_seconds + (open_browser_result.duration_seconds if open_browser_result else 0) + (desktop_screenshot_result.duration_seconds if desktop_screenshot_result else 0), 3, ), "model_duration_seconds": round( self._model_duration(host_decision, app_decision) + float(model_analysis.get("model_duration_seconds") or 0), 3, ), "hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(), "host_decision": host_decision, "app_decision": app_decision, "model_available": self.model.available, "model_analysis": model_analysis, "model_errors": self._model_errors(host_decision, app_decision, model_analysis), "observation": self._observation_summary(observation), "browser_open": open_browser_result.result if open_browser_result else None, "screenshots": [ desktop_screenshot_result.result ] if desktop_screenshot_result and desktop_screenshot_result.ok else [], "available_tools": self.tools.descriptions(), "discovered_child_tasks": self._child_task_briefs(discovered_children), "model_generated_child_tasks": self._child_task_briefs(model_generated_children), "rule_discovered_child_tasks": self._child_task_briefs(rule_discovered_children), "discovery_source": discovery_source, "submitted_child_tasks": submitted_children, "memory": self.blackboard.summary(), "logs": [*self.logs, *(observation_result.result.get("logs") or [])], } self.blackboard.append( "trajectories", { "host_decision": host_decision, "app_decision": app_decision, "result": { "success": success, "message": message, "submitted_child_count": len(submitted_children), "duration_seconds": duration, }, }, task_id=self.task_id, ) result["memory"] = self.blackboard.summary() return result def execution_mode(self) -> str: explicit = str(self.payload.get("execution_mode") or self.payload.get("agent_mode") or "").strip().lower() if explicit in {"web", "gui"}: return explicit task_type = str(self.payload.get("task_type") or "").lower() if task_type.startswith("gui") or self.payload.get("window_id") or self.payload.get("window_title"): return "gui" return "web" def _run_gui_task(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]: windows_result = self.tools.gui.list_windows() screenshot_result = self.tools.gui.capture_desktop(self.screenshot_dir, prefix=f"task_{self.task.get('id') or self.task_id}_desktop") selected_window = self._select_window(windows_result.result.get("windows") or []) inspect_result: ToolResult | None = None window_screenshot_result: ToolResult | None = None if selected_window: inspect_result = self.tools.gui.inspect_window(str(selected_window["id"])) window_screenshot_result = self.tools.gui.capture_window( str(selected_window["id"]), self.screenshot_dir, prefix=f"task_{self.task.get('id') or self.task_id}_window", ) observation = self._gui_observation_summary( windows_result, screenshot_result, selected_window, inspect_result, window_screenshot_result, ) self.blackboard.append( "observations", { "tool": "gui_observe", "ok": windows_result.ok, "error": windows_result.error, "summary": observation, }, task_id=self.task_id, ) if screenshot_result.ok: self.blackboard.append( "screenshots", {"image_path": screenshot_result.result.get("image_path"), "scope": "desktop"}, task_id=self.task_id, ) if window_screenshot_result and window_screenshot_result.ok: self.blackboard.append( "screenshots", {"image_path": window_screenshot_result.result.get("image_path"), "scope": "window", "window": selected_window}, task_id=self.task_id, ) app_decision = { "observation": json.dumps(observation, ensure_ascii=False), "thought": "GUI P0 阶段只枚举窗口、控件并保存证据截图，不执行点击或输入。", "action": { "function": "list_windows" if not selected_window else "inspect_window_controls", "arguments": { "window_id": selected_window.get("id") if selected_window else None, "window_title": self.payload.get("window_title"), }, "status": "FINISH" if windows_result.ok and (not self.payload.get("window_title") or selected_window) else "FAIL", }, "plan": [], "comment": "GUI 观察完成。" if windows_result.ok else windows_result.error, "save_screenshot": { "save": bool(screenshot_result.ok or (window_screenshot_result and window_screenshot_result.ok)), "reason": "GUI P0 观察需要截图作为证据。" if screenshot_result.ok else screenshot_result.error, }, "result": "GUI 窗口和控件信息已记录。" if windows_result.ok else windows_result.error, "model_used": False, } success = windows_result.ok and (not self.payload.get("window_title") or bool(selected_window)) message = "Agent GUI P0 observation completed." if success else "Requested GUI window was not found or could not be inspected." duration = round(time.monotonic() - started, 3) result = { "success": success, "message": message, "task_type": self.payload.get("task_type") or "gui_p0", "agent_framework": "ufo-inspired-lite", "p0_mode": True, "execution_mode": "gui", "needs_user_input": False, "duration_seconds": duration, "action_duration_seconds": round( windows_result.duration_seconds + screenshot_result.duration_seconds + (inspect_result.duration_seconds if inspect_result else 0) + (window_screenshot_result.duration_seconds if window_screenshot_result else 0), 3, ), "model_duration_seconds": self._model_duration(host_decision, app_decision), "hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(), "host_decision": host_decision, "app_decision": app_decision, "observation": observation, "available_tools": self.tools.descriptions(), "memory": self.blackboard.summary(), "logs": [ *self.logs, *(windows_result.error and [windows_result.error] or []), *(screenshot_result.error and [screenshot_result.error] or []), *(inspect_result and inspect_result.error and [inspect_result.error] or []), ], } self.blackboard.append( "trajectories", { "host_decision": host_decision, "app_decision": app_decision, "result": { "success": success, "message": message, "duration_seconds": duration, }, }, task_id=self.task_id, ) result["memory"] = self.blackboard.summary() return result def _host_decision(self) -> dict[str, Any]: fallback = { "observation": "收到一个主控端分配的测试任务。", "thought": "P0 阶段先把任务限定到单个 Web/AppAgent，执行无副作用观察，并按页面结构生成后续子任务。", "current_subtask": self.payload.get("objective") or self.payload.get("instruction") or self.task.get("title") or "", "message": ["只执行无副作用观察动作；遇到登录、验证码、权限或破坏性入口时暂停并回传。"], "status": "ASSIGN", "plan": ["观察目标页面", "记录页面元信息", "识别可继续拆分的链接、表单和入口"], "function": "assign_app_agent", "arguments": {"agent": "web_app_agent", "target_url": self.target_url()}, "questions": [], "result": "已分配给 Web AppAgent 执行 P0 观察。", "model_used": False, } if not self.model.available: return fallback context = { "task": self._task_brief(), "blackboard": self.blackboard.snapshot(limit=5), "tools": self.tools.descriptions(), } response = self.model.chat_json( [ {"role": "system", "content": HOST_AGENT_SYSTEM_PROMPT}, {"role": "user", "content": json.dumps(context, ensure_ascii=False)}, ] ) if response.ok and isinstance(response.parsed, dict): parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds} return parsed self.logs.append(f"HostAgent model fallback: {response.error or 'invalid JSON response'}") return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds} def _app_decision(self, host_decision: dict[str, Any], observation_result: ToolResult) -> dict[str, Any]: observation = self._observation_summary(observation_result.result) fallback_status = "FINISH" if observation_result.ok else "FAIL" if observation.get("requires_auth"): fallback_status = "PENDING" fallback = { "observation": json.dumps(observation, ensure_ascii=False), "thought": "P0 阶段采用 observe_web_page 工具完成可达性和页面结构观察，不执行提交、保存、删除等有副作用动作。", "action": { "function": "observe_web_page", "arguments": {"target_url": self.target_url()}, "status": fallback_status, }, "plan": [], "comment": "页面观察完成。" if observation_result.ok else observation_result.error, "save_screenshot": {"save": False, "reason": "P0 Web 观察暂不保存截图。"}, "result": "页面可达性和结构信息已记录。" if observation_result.ok else observation_result.error, "model_used": False, } if not self.model.available: return fallback context = { "host_decision": host_decision, "task": self._task_brief(), "observation": observation, "available_tools": self.tools.descriptions(), "recent_trajectories": self.blackboard.recent("trajectories", 5), } response = self.model.chat_json( [ {"role": "system", "content": APP_AGENT_SYSTEM_PROMPT}, {"role": "user", "content": json.dumps(context, ensure_ascii=False)}, ] ) if response.ok and isinstance(response.parsed, dict): parsed = {**fallback, **response.parsed, "model_used": True, "model_duration_seconds": response.duration_seconds} return parsed self.logs.append(f"AppAgent model fallback: {response.error or 'invalid JSON response'}") return {**fallback, "model_error": response.error, "model_duration_seconds": response.duration_seconds} def _model_discover_child_tasks(self, observation_result: ToolResult) -> tuple[dict[str, Any], list[dict[str, Any]]]: base_analysis: dict[str, Any] = { "model_available": self.model.available, "model_called": False, "model_call_succeeded": False, "model_used": False, "model_duration_seconds": 0, "summary": "", "coverage_notes": [], "needs_user_input": False, "user_input_reason": "", "functional_task_count": 0, } if not observation_result.ok: return {**base_analysis, "error": observation_result.error or "Page observation failed."}, [] if not self.model.available: self.logs.append("Model config is incomplete; used rule-based discovery fallback.") return {**base_analysis, "error": "Model config is incomplete."}, [] observation = observation_result.result context = { "task": self._task_brief(), "hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(), "page": self._observation_for_model(observation), "recent_memory": self.blackboard.snapshot(limit=4), "constraints": { "p0_no_side_effects": True, "max_functional_tasks": 8, "discovery_max_depth": int(self.payload.get("discovery_max_depth") or 2), }, } response = self.model.chat_json( [ {"role": "system", "content": SYSTEM_TEST_ANALYST_PROMPT}, {"role": "user", "content": json.dumps(context, ensure_ascii=False)}, ] ) if not response.ok or not isinstance(response.parsed, dict): self.logs.append(f"SystemTestAnalyst model fallback: {response.error or 'invalid JSON response'}") return { **base_analysis, "model_called": True, "model_call_succeeded": False, "model_used": False, "model_duration_seconds": response.duration_seconds, "error": response.error or "Model did not return a valid JSON object.", }, [] parsed = response.parsed children = self._children_from_model_analysis(parsed, observation) analysis = { **base_analysis, "model_called": True, "model_call_succeeded": True, "model_used": True, "model_duration_seconds": response.duration_seconds, "summary": str(parsed.get("summary") or ""), "coverage_notes": parsed.get("coverage_notes") if isinstance(parsed.get("coverage_notes"), list) else [], "needs_user_input": bool(parsed.get("needs_user_input")), "user_input_reason": str(parsed.get("user_input_reason") or ""), "functional_task_count": len(children), } self.logs.append( "Model analyzed page and generated " f"{len(children)} functional task(s): {analysis.get('summary') or 'no summary'}" ) return analysis, children def _observation_for_model(self, observation: dict[str, Any]) -> dict[str, Any]: def clean_items(items: Any, limit: int) -> list[dict[str, Any]]: if not isinstance(items, list): return [] cleaned: list[dict[str, Any]] = [] for item in items[:limit]: if not isinstance(item, dict): continue cleaned.append( { key: value for key, value in item.items() if key in {"href", "text", "action", "method", "id", "name", "class", "type", "fields", "placeholder", "aria_label", "in_form"} } ) return cleaned return { "url": observation.get("url"), "final_url": observation.get("final_url"), "http_status": observation.get("http_status"), "content_type": observation.get("content_type"), "page_title": observation.get("page_title"), "requires_auth": observation.get("requires_auth"), "has_login_form": observation.get("has_login_form"), "links": clean_items(observation.get("links"), 20), "forms": clean_items(observation.get("forms"), 12), "buttons": clean_items(observation.get("buttons"), 20), } def _children_from_model_analysis(self, analysis: dict[str, Any], observation: dict[str, Any]) -> list[dict[str, Any]]: raw_items = analysis.get("functional_tasks") if not isinstance(raw_items, list): return [] max_children = 8 children: list[dict[str, Any]] = [] seen: set[tuple[str, str]] = set() for index, item in enumerate(raw_items[:max_children], start=1): if not isinstance(item, dict): continue child = self._child_from_model_item(item, observation, index) key = ( str(child.get("title") or "").strip().lower(), str((child.get("payload") or {}).get("target_url") or "").strip().lower(), ) if key in seen: continue seen.add(key) children.append(child) return children def _child_from_model_item(self, item: dict[str, Any], observation: dict[str, Any], index: int) -> dict[str, Any]: parent_external_id = str(self.task.get("external_id") or self.task.get("id") or "") root_external_id = str(self.task.get("root_external_id") or self.payload.get("root_external_id") or parent_external_id) parent_depth = int(self.task.get("depth") or self.payload.get("depth") or 0) depth = parent_depth + 1 discovery_max_depth = int(self.payload.get("discovery_max_depth") or 2) base_url = observation.get("final_url") or observation.get("url") or self.target_url() raw_target_url = str(item.get("target_url") or "").strip() target_url = urljoin(base_url, raw_target_url) if raw_target_url else base_url title = str(item.get("title") or f"模型生成测试任务 {index}").strip() objective = str(item.get("objective") or title).strip() module = str(item.get("module") or title).strip() level = str(item.get("level") or "feature").strip().lower() if level not in {"module", "feature", "case"}: level = "feature" risk = str(item.get("risk_level") or "low").strip().lower() if risk not in {"low", "medium", "high"}: risk = "low" try: priority = int(item.get("priority") or 30 + depth * 10 + index) except (TypeError, ValueError): priority = 30 + depth * 10 + index preconditions = item.get("preconditions") if isinstance(item.get("preconditions"), list) else [] external_id = self._stable_child_id(root_external_id, parent_external_id, title, target_url, objective) payload = { "task_type": "smart_discovery_p0", "instruction": objective, "objective": objective, "target_url": target_url, "source": "llm_analysis", "llm_reason": item.get("reason") or "", "level": level, "depth": depth, "risk_level": risk, "allow_child_discovery": level in {"module", "feature"} and depth < discovery_max_depth, "discovery_max_depth": discovery_max_depth, "open_in_browser": True, "capture_screenshot": True, "preconditions": preconditions, "hierarchy": { "id": external_id, "parent_id": parent_external_id, "root_id": root_external_id, "depth": depth, "level": level, }, } return { "external_id": external_id, "title": title, "module": module, "parent_external_id": parent_external_id, "root_external_id": root_external_id, "depth": depth, "task_level": level, "risk_level": risk, "priority": priority, "failure_policy": "pause" if risk in {"medium", "high"} or preconditions else "skip", "max_retries": 2, "timeout_seconds": 90, "tags": ["llm-generated", "system-test", "p0"], "payload": payload, } def _stable_child_id(self, *parts: Any) -> str: raw = "|".join(str(part or "") for part in parts) return f"llm-{hashlib.sha1(raw.encode('utf-8')).hexdigest()[:12]}" def _allow_child_discovery(self, observation_result: ToolResult) -> bool: if not observation_result.ok: return False allow = self.payload.get("allow_child_discovery") if allow is not None: return bool(allow) if self.payload.get("task_type") != "smart_discovery_p0": return False level = str(self.payload.get("level") or self.task.get("task_level") or "") depth = int(self.payload.get("depth") or self.task.get("depth") or 0) return level in {"site", "module", ""} and depth < int(self.payload.get("discovery_max_depth") or 2) def _submit_child_tasks(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]: if not children or self.submit_task is None: return [] submitted: list[dict[str, Any]] = [] plan_name = self.task.get("plan_name") or self.payload.get("plan_name") or "智能探索 P0" for child in children: payload = {**child, "plan_name": plan_name} try: created = self.submit_task(payload) except Exception as exc: self.logs.append(f"Submit child task failed: {child.get('external_id')} error={exc}") continue submitted.append( { "external_id": created.get("external_id") or child.get("external_id"), "title": created.get("title") or child.get("title"), "depth": created.get("depth") if created.get("depth") is not None else child.get("depth"), "task_level": created.get("task_level") or child.get("task_level"), } ) return submitted def _status_from_observation(self, observation_result: ToolResult, app_decision: dict[str, Any]) -> tuple[bool, str, bool]: action = app_decision.get("action") if isinstance(app_decision.get("action"), dict) else {} status = str(action.get("status") or "").upper() if status == "PENDING": return False, "Task needs user input before it can continue.", True if status in {"FAIL", "CONFIRM"}: return False, app_decision.get("result") or app_decision.get("comment") or "Agent action did not complete.", status == "CONFIRM" if not observation_result.ok: return False, observation_result.error or "Target observation failed.", False if observation_result.result.get("http_status") in {401, 403}: return False, "Target page requires authentication or permission.", True return True, "Agent P0 observation completed.", False def _finish_without_target(self, started: float, host_decision: dict[str, Any]) -> dict[str, Any]: task_type = self.payload.get("task_type") or "manual" success = task_type != "smart_discovery_p0" duration = round(time.monotonic() - started, 3) result = { "success": success, "message": "No executable target URL was provided." if not success else "Task accepted; no executable URL was provided for P0 mode.", "task_type": task_type, "agent_framework": "ufo-inspired-lite", "p0_mode": True, "needs_user_input": not success, "duration_seconds": duration, "hierarchy": self.payload.get("hierarchy") or self._task_hierarchy(), "host_decision": host_decision, "app_decision": { "action": {"function": "no_action", "arguments": {}, "status": "FAIL" if not success else "FINISH"}, "result": "Missing target URL.", }, "available_tools": self.tools.descriptions(), "memory": self.blackboard.summary(), "logs": ["AgentSession requires target_url for Web P0 execution."], } self.blackboard.append("trajectories", {"host_decision": host_decision, "result": result}, task_id=self.task_id) return result def _select_window(self, windows: list[dict[str, Any]]) -> dict[str, Any] | None: window_id = self.payload.get("window_id") if window_id: for window in windows: if str(window.get("id")) == str(window_id) or str(window.get("hex_id")).lower() == str(window_id).lower(): return window return None title = str(self.payload.get("window_title") or self.payload.get("application") or "").strip().lower() if not title: return None for window in windows: if title in str(window.get("title") or "").lower(): return window for window in windows: if title in str(window.get("class_name") or "").lower(): return window return None def _gui_observation_summary( self, windows_result: ToolResult, screenshot_result: ToolResult, selected_window: dict[str, Any] | None, inspect_result: ToolResult | None, window_screenshot_result: ToolResult | None, ) -> dict[str, Any]: windows = windows_result.result.get("windows") if isinstance(windows_result.result, dict) else [] controls = [] if inspect_result and isinstance(inspect_result.result, dict): controls = inspect_result.result.get("controls") or [] return { "backend": (windows_result.result or {}).get("backend") or self.tools.gui.backend_status(), "window_count": len(windows or []), "windows": [ { "id": window.get("id"), "title": window.get("title"), "class_name": window.get("class_name"), "process_id": window.get("process_id"), "rect": window.get("rect"), "minimized": window.get("minimized"), } for window in (windows or [])[:20] ], "selected_window": selected_window, "control_count": len(controls), "controls": [ { "id": control.get("id"), "text": control.get("text"), "class_name": control.get("class_name"), "rect": control.get("rect"), "visible": control.get("visible"), "enabled": control.get("enabled"), } for control in controls[:80] ], "desktop_screenshot": screenshot_result.result if screenshot_result.ok else None, "window_screenshot": window_screenshot_result.result if window_screenshot_result and window_screenshot_result.ok else None, "errors": [ message for message in [ windows_result.error, screenshot_result.error, inspect_result.error if inspect_result else "", window_screenshot_result.error if window_screenshot_result else "", ] if message ], } def _task_brief(self) -> dict[str, Any]: return { "id": self.task.get("id"), "external_id": self.task.get("external_id"), "title": self.task.get("title"), "plan_name": self.task.get("plan_name"), "module": self.task.get("module"), "status": self.task.get("status"), "depth": self.task.get("depth"), "task_level": self.task.get("task_level"), "risk_level": self.task.get("risk_level"), "payload": self.payload, } def _task_hierarchy(self) -> dict[str, Any]: return { "id": self.task.get("external_id") or self.task.get("id"), "parent_id": self.task.get("parent_external_id"), "root_id": self.task.get("root_external_id") or self.task.get("external_id") or self.task.get("id"), "depth": self.task.get("depth") or 0, "level": self.task.get("task_level") or self.payload.get("level") or "", } def _observation_summary(self, observation: dict[str, Any]) -> dict[str, Any]: return { "url": observation.get("url"), "final_url": observation.get("final_url"), "http_status": observation.get("http_status"), "content_type": observation.get("content_type"), "page_title": observation.get("page_title"), "requires_auth": observation.get("requires_auth"), "auth_challenge": observation.get("auth_challenge"), "has_login_form": observation.get("has_login_form"), "link_count": len(observation.get("links") or []), "form_count": len(observation.get("forms") or []), "button_count": len(observation.get("buttons") or []), } def _child_task_briefs(self, children: list[dict[str, Any]]) -> list[dict[str, Any]]: return [ { "external_id": child.get("external_id"), "title": child.get("title"), "parent_external_id": child.get("parent_external_id"), "root_external_id": child.get("root_external_id"), "depth": child.get("depth"), "task_level": child.get("task_level"), "risk_level": child.get("risk_level"), "target_url": (child.get("payload") or {}).get("target_url"), "source": (child.get("payload") or {}).get("source"), "preconditions": (child.get("payload") or {}).get("preconditions") or [], "llm_reason": (child.get("payload") or {}).get("llm_reason") or "", } for child in children ] def _model_duration(self, host_decision: dict[str, Any], app_decision: dict[str, Any]) -> float: return round( float(host_decision.get("model_duration_seconds") or 0) + float(app_decision.get("model_duration_seconds") or 0), 3, ) def _model_errors(self, *parts: dict[str, Any]) -> list[str]: errors: list[str] = [] for part in parts: error = str(part.get("model_error") or part.get("error") or "").strip() if error and error not in errors: errors.append(error) return errors