from __future__ import annotations import hashlib import json import html import ipaddress import mimetypes import os import re import socket import ssl import sys import threading import webbrowser from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from html.parser import HTMLParser from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError from urllib.parse import parse_qs, urldefrag, urljoin, urlparse from urllib.request import Request, urlopen from . import __version__ from .config import ControllerConfig from .storage import AlreadyRegisteredError, Database STARTED_AT = datetime.now(timezone.utc).isoformat(timespec="seconds") DEFAULT_SITE_SCAN_PORTS = [ 80, 443, 3000, 4173, 5173, 5174, 8000, 8001, 8080, 8081, 8088, 8765, 9000, 9090, 10000, ] TLS_SITE_PORTS = {443, 8443, 9443} MAX_SITE_SCAN_PROBES = 512 MAX_SITE_SCAN_PORTS = 256 DISCOVERY_MAX_LINKS = 24 DISCOVERY_MAX_FORMS = 12 DISCOVERY_MAX_BUTTONS = 24 DISCOVERY_MAX_HTML_BYTES = 1_500_000 TITLE_RE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) UNVERIFIED_HTTPS_CONTEXT = ssl._create_unverified_context() def _browser_autostart_enabled() -> bool: disabled_values = {"1", "true", "yes", "on"} return ( os.environ.get("CONTROLLER_NO_BROWSER", "").strip().lower() not in disabled_values and os.environ.get("AGENT_CONTROLLER_NO_BROWSER", "").strip().lower() not in disabled_values ) def _open_panel_later(url: str) -> None: def open_panel() -> None: try: webbrowser.open(url, new=2) except Exception as exc: # Browser launch should not bring the service down. print(f"Could not open browser automatically: {exc}", file=sys.stderr) threading.Timer(0.8, open_panel).start() def _json_bytes(payload: Any) -> bytes: return json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8") def _chat_completions_url(base_url: str) -> str: base = base_url.rstrip("/") if base.endswith("/chat/completions"): return base if base.endswith("/v1"): return f"{base}/chat/completions" return f"{base}/v1/chat/completions" def _parse_json_object(text: str) -> dict[str, Any] | None: raw = text.strip() fence = re.search(r"```(?:json)?\s*(.*?)```", raw, re.IGNORECASE | re.DOTALL) if fence: raw = fence.group(1).strip() start = raw.find("{") end = raw.rfind("}") if start >= 0 and end > start: raw = raw[start : end + 1] try: parsed = json.loads(raw) except json.JSONDecodeError: return None return parsed if isinstance(parsed, dict) else None def _completion_content(data: dict[str, Any]) -> str: choices = data.get("choices") if not isinstance(choices, list) or not choices: return "" choice = choices[0] if isinstance(choices[0], dict) else {} message = choice.get("message") if isinstance(choice.get("message"), dict) else {} content = ( message.get("content") or message.get("reasoning_content") or message.get("reasoning") or choice.get("text") or "" ) if not isinstance(content, str): return json.dumps(content, ensure_ascii=False) return content def _port_is_available(host: str, port: int) -> bool: probe_host = "127.0.0.1" if host in {"0.0.0.0", "::", ""} else host with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.settimeout(0.5) return sock.connect_ex((probe_host, port)) != 0 def _is_blank(value: Any) -> bool: return value is None or value == [] or (isinstance(value, str) and not value.strip()) def _parse_site_scan_ports(value: Any) -> list[int]: if _is_blank(value): return list(DEFAULT_SITE_SCAN_PORTS) raw_parts = value if isinstance(value, list) else str(value).replace(";", ",").replace("，", ",").split(",") ports: set[int] = set() for raw_part in raw_parts: text = str(raw_part).strip() if not text: continue if "-" in text: start_text, end_text = [part.strip() for part in text.split("-", 1)] if not start_text.isdigit() or not end_text.isdigit(): raise ValueError(f"Invalid port range: {text}") start, end = int(start_text), int(end_text) if end < start: raise ValueError(f"Invalid port range: {text}") if end - start + 1 > MAX_SITE_SCAN_PORTS: raise ValueError("Port range is too large; narrow it before scanning.") candidates = range(start, end + 1) else: if not text.isdigit(): raise ValueError(f"Invalid port: {text}") candidates = [int(text)] for port in candidates: if port < 1 or port > 65535: raise ValueError(f"Port out of range: {port}") ports.add(port) if not ports: raise ValueError("No valid ports to scan.") if len(ports) > MAX_SITE_SCAN_PORTS: raise ValueError("Too many ports to scan at once; please narrow the port list.") return sorted(ports) def _resolve_ipv4(host: str) -> str | None: try: infos = socket.getaddrinfo(host, None, socket.AF_INET, socket.SOCK_STREAM) except socket.gaierror: return None for info in infos: address = info[4][0] if address: return address return None def _add_local_ipv4(candidates: list[str], address: str | None) -> None: if not address or address in candidates: return try: parsed = ipaddress.ip_address(address) except ValueError: return if parsed.version != 4 or parsed.is_unspecified or parsed.is_multicast: return candidates.append(address) def _local_site_scan_hosts() -> list[dict[str, str]]: addresses = ["127.0.0.1"] try: for address in socket.gethostbyname_ex(socket.gethostname())[2]: _add_local_ipv4(addresses, address) except OSError: pass try: with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: sock.connect(("8.8.8.8", 80)) _add_local_ipv4(addresses, sock.getsockname()[0]) except OSError: pass return [{"host": address, "ip": address} for address in addresses] def _site_scan_hosts(target: str) -> list[dict[str, str]]: normalized = target.strip() if normalized.lower() in {"", "local", "localhost", "127.0.0.1", "::1"}: return _local_site_scan_hosts() if any(marker in normalized for marker in ["/", "\\", ",", ";", " "]): raise ValueError("Please enter one IP address or host name, not a network range.") ip = _resolve_ipv4(normalized) if ip is None: raise ValueError(f"Unable to resolve host: {normalized}") return [{"host": normalized, "ip": ip}] def _extract_site_scan_target(payload: dict[str, Any]) -> tuple[str, Any]: target = str(payload.get("host") or "").strip() ports = payload.get("ports") if "://" in target: parsed = urlparse(target) if parsed.hostname: target = parsed.hostname if _is_blank(ports) and parsed.port: ports = parsed.port elif target.count(":") == 1: host_part, port_part = target.rsplit(":", 1) if host_part.strip() and port_part.isdigit(): target = host_part.strip() if _is_blank(ports): ports = port_part return target, ports def _scheme_candidates(port: int) -> list[str]: return ["https", "http"] if port in TLS_SITE_PORTS else ["http", "https"] def _decode_body(raw: bytes, content_type: str) -> str: charset = "utf-8" match = re.search(r"charset=([^;\s]+)", content_type or "", re.IGNORECASE) if match: charset = match.group(1).strip("\"'") for candidate in [charset, "utf-8", "gbk", "latin-1"]: try: return raw.decode(candidate, errors="replace") except LookupError: continue return raw.decode("utf-8", errors="replace") def _extract_page_title(raw: bytes, content_type: str) -> str: text = _decode_body(raw, content_type) match = TITLE_RE.search(text) if not match: return "" return html.unescape(re.sub(r"\s+", " ", match.group(1))).strip() def _clean_discovery_text(value: Any, max_length: int = 160) -> str: text = html.unescape(re.sub(r"\s+", " ", str(value or ""))).strip() return text[:max_length].strip() def _normalize_discovery_url(value: Any) -> str: text = str(value or "").strip() if not text: raise ValueError("Missing target system URL.") if "://" not in text: text = f"http://{text}" parsed = urlparse(text) if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ValueError("Target system URL must be an http(s) URL.") return text def _origin_tuple(value: str) -> tuple[str, str, int]: parsed = urlparse(value) port = parsed.port or (443 if parsed.scheme == "https" else 80) return parsed.scheme, (parsed.hostname or "").lower(), port def _same_origin(base_url: str, candidate_url: str) -> bool: try: return _origin_tuple(base_url) == _origin_tuple(candidate_url) except ValueError: return False def _safe_join_page_url(base_url: str, href: str) -> str | None: text = str(href or "").strip() if not text or text.startswith(("#", "javascript:", "mailto:", "tel:", "data:")): return None joined = urljoin(base_url, text) joined, _fragment = urldefrag(joined) parsed = urlparse(joined) if parsed.scheme not in {"http", "https"} or not parsed.netloc: return None if not _same_origin(base_url, joined): return None return joined class DiscoveryHtmlParser(HTMLParser): def __init__(self) -> None: super().__init__(convert_charrefs=True) self.title_parts: list[str] = [] self.links: list[dict[str, Any]] = [] self.forms: list[dict[str, Any]] = [] self.buttons: list[dict[str, Any]] = [] self._in_title = False self._anchor_stack: list[dict[str, Any]] = [] self._current_form: dict[str, Any] | None = None self._current_button: dict[str, Any] | None = None def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: tag = tag.lower() attr = {name.lower(): value or "" for name, value in attrs} if tag == "title": self._in_title = True return if tag == "a": self._anchor_stack.append({"href": attr.get("href", ""), "text_parts": []}) return if tag == "form": self._current_form = { "action": attr.get("action", ""), "method": (attr.get("method") or "get").upper(), "id": attr.get("id", ""), "name": attr.get("name", ""), "class": attr.get("class", ""), "text_parts": [], "fields": [], } return if self._current_form is not None and tag in {"input", "select", "textarea"}: self._current_form["fields"].append( { "tag": tag, "type": (attr.get("type") or tag).lower(), "name": attr.get("name", ""), "id": attr.get("id", ""), "placeholder": attr.get("placeholder", ""), "aria_label": attr.get("aria-label", ""), } ) if tag == "button": self._current_button = { "type": attr.get("type", ""), "id": attr.get("id", ""), "name": attr.get("name", ""), "class": attr.get("class", ""), "text_parts": [], "in_form": self._current_form is not None, } def handle_data(self, data: str) -> None: if self._in_title: self.title_parts.append(data) if self._anchor_stack: self._anchor_stack[-1]["text_parts"].append(data) if self._current_form is not None: self._current_form["text_parts"].append(data) if self._current_button is not None: self._current_button["text_parts"].append(data) def handle_endtag(self, tag: str) -> None: tag = tag.lower() if tag == "title": self._in_title = False elif tag == "a" and self._anchor_stack: anchor = self._anchor_stack.pop() self.links.append( { "href": anchor.get("href", ""), "text": _clean_discovery_text(" ".join(anchor.get("text_parts") or []), 120), } ) elif tag == "button" and self._current_button is not None: button = self._current_button button["text"] = _clean_discovery_text(" ".join(button.get("text_parts") or []), 120) self.buttons.append(button) self._current_button = None elif tag == "form" and self._current_form is not None: form = self._current_form form["text"] = _clean_discovery_text(" ".join(form.get("text_parts") or []), 160) self.forms.append(form) self._current_form = None def _fetch_discovery_page(url: str, timeout: float) -> dict[str, Any]: request = Request( url, headers={ "Accept": "text/html,application/xhtml+xml,*/*", "User-Agent": f"SystemTestController/{__version__} discovery", }, ) started = datetime.now(timezone.utc) try: with urlopen(request, timeout=timeout, context=UNVERIFIED_HTTPS_CONTEXT) as response: raw = response.read(DISCOVERY_MAX_HTML_BYTES) content_type = response.headers.get("Content-Type", "") final_url = response.geturl() status = response.status except HTTPError as exc: raw = exc.read(DISCOVERY_MAX_HTML_BYTES) content_type = exc.headers.get("Content-Type", "") final_url = exc.geturl() status = exc.code except (URLError, TimeoutError, OSError) as exc: reason = getattr(exc, "reason", exc) if isinstance(exc, TimeoutError) or isinstance(reason, TimeoutError): reason = f"timed out after {timeout:g}s" raise ValueError(f"Target system is not reachable: {reason}") from exc elapsed = (datetime.now(timezone.utc) - started).total_seconds() return { "url": url, "final_url": final_url, "status": status, "content_type": content_type, "html": _decode_body(raw, content_type), "elapsed_seconds": round(elapsed, 3), } def _candidate_id(seed: str, *parts: Any) -> str: raw = "|".join([seed, *[str(part or "") for part in parts]]) digest = hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12] return f"auto-{digest}" def _risk_for_text(text: str) -> str: lowered = text.lower() high = ["delete", "remove", "drop", "支付", "付款", "删除", "注销", "清空", "退款"] medium = ["save", "submit", "create", "update", "edit", "upload", "保存", "提交", "新增", "修改", "上传"] if any(word in lowered for word in high): return "high" if any(word in lowered for word in medium): return "medium" return "low" def _looks_like_login_form(form: dict[str, Any]) -> bool: fields = form.get("fields") or [] text = " ".join( [ str(form.get("id") or ""), str(form.get("name") or ""), str(form.get("class") or ""), str(form.get("text") or ""), *[ " ".join( [ str(field.get("type") or ""), str(field.get("name") or ""), str(field.get("id") or ""), str(field.get("placeholder") or ""), str(field.get("aria_label") or ""), ] ) for field in fields ], ] ).lower() return "password" in text or any(keyword in text for keyword in ["login", "signin", "sign in", "登录", "账号", "密码"]) def _build_discovery_candidates(target_url: str, page: dict[str, Any]) -> dict[str, Any]: parser = DiscoveryHtmlParser() try: parser.feed(page.get("html") or "") except Exception: pass page_url = page.get("final_url") or target_url page_title = _clean_discovery_text(" ".join(parser.title_parts), 120) or _extract_page_title( str(page.get("html") or "").encode("utf-8", errors="ignore"), "text/html; charset=utf-8", ) display_name = page_title or urlparse(page_url).netloc or page_url root_id = _candidate_id(page_url, "root", display_name) candidates: list[dict[str, Any]] = [ { "id": root_id, "parent_id": None, "root_id": root_id, "depth": 0, "level": "site", "title": f"站点冒烟检查：{display_name}", "objective": "确认待测站点可以访问，并记录页面标题、状态码和基础元信息。", "target_url": page_url, "module": "站点入口", "risk_level": "low", "priority": 20, "max_retries": 2, "failure_policy": "pause", "preconditions": [], "source": "page", "tags": ["smart-discovery", "p0", "site"], "selected": True, } ] seen_links: set[str] = set() for link in parser.links: if len(seen_links) >= DISCOVERY_MAX_LINKS: break href = _safe_join_page_url(page_url, str(link.get("href") or "")) if not href or href in seen_links: continue seen_links.add(href) label = _clean_discovery_text(link.get("text"), 80) or urlparse(href).path.strip("/") or href risk = _risk_for_text(label) candidate_id = _candidate_id(page_url, "link", href, label) candidates.append( { "id": candidate_id, "parent_id": root_id, "root_id": root_id, "depth": 1, "level": "module", "title": f"探索页面：{label}", "objective": f"访问并检查页面“{label}”是否可达，记录页面标题和返回状态。", "target_url": href, "module": label, "risk_level": risk, "priority": 40 if risk == "low" else 70, "max_retries": 2, "failure_policy": "pause" if risk == "high" else "skip", "preconditions": [], "source": "html_link", "tags": ["smart-discovery", "p0", "navigation"], "selected": risk != "high", } ) for index, form in enumerate(parser.forms[:DISCOVERY_MAX_FORMS], start=1): action_url = _safe_join_page_url(page_url, str(form.get("action") or "")) or page_url is_login = _looks_like_login_form(form) field_names = [ _clean_discovery_text( field.get("placeholder") or field.get("aria_label") or field.get("name") or field.get("id") or field.get("type"), 40, ) for field in form.get("fields") or [] ] field_names = [name for name in field_names if name][:8] label = "登录表单" if is_login else (_clean_discovery_text(form.get("name") or form.get("id") or form.get("text"), 60) or f"表单 {index}") risk = "medium" if is_login else _risk_for_text(label) candidate_id = _candidate_id(page_url, "form", index, action_url, label) preconditions = ["需要测试账号/密码"] if is_login else [] candidates.append( { "id": candidate_id, "parent_id": root_id, "root_id": root_id, "depth": 1, "level": "feature", "title": f"识别功能：{label}", "objective": f"分析{label}的输入项与提交入口，P0 阶段先确认页面元素存在，不执行真实提交。", "target_url": action_url, "module": label, "risk_level": risk, "priority": 35 if is_login else 55, "max_retries": 2, "failure_policy": "pause" if is_login else "skip", "preconditions": preconditions, "source": "html_form", "method": form.get("method") or "GET", "fields": field_names, "tags": ["smart-discovery", "p0", "form"], "selected": True, } ) seen_buttons: set[str] = set() for button in parser.buttons: if len(seen_buttons) >= DISCOVERY_MAX_BUTTONS: break label = _clean_discovery_text(button.get("text") or button.get("name") or button.get("id"), 60) if not label or label in seen_buttons: continue seen_buttons.add(label) risk = _risk_for_text(label) candidate_id = _candidate_id(page_url, "button", label) candidates.append( { "id": candidate_id, "parent_id": root_id, "root_id": root_id, "depth": 1, "level": "feature", "title": f"识别入口：{label}", "objective": f"确认入口“{label}”在页面中存在；高风险入口仅记录，不自动执行破坏性动作。", "target_url": page_url, "module": "页面入口", "risk_level": risk, "priority": 60 if risk == "low" else 85, "max_retries": 1, "failure_policy": "pause" if risk == "high" else "skip", "preconditions": [], "source": "html_button", "tags": ["smart-discovery", "p0", "entry"], "selected": risk != "high", } ) if len(candidates) == 1: for label, objective in [ ("单页应用基础冒烟测试", "确认单页应用根页面可达，并为后续 GUI 探索提供初始页面。"), ("页面导航与菜单发现", "后续由 GUI Agent 在页面内识别菜单、按钮和可交互区域。"), ]: candidate_id = _candidate_id(page_url, "fallback", label) candidates.append( { "id": candidate_id, "parent_id": root_id, "root_id": root_id, "depth": 1, "level": "module", "title": label, "objective": objective, "target_url": page_url, "module": "页面探索", "risk_level": "low", "priority": 50, "max_retries": 2, "failure_policy": "skip", "preconditions": [], "source": "fallback", "tags": ["smart-discovery", "p0", "spa"], "selected": True, } ) return { "summary": { "target_url": target_url, "final_url": page_url, "page_title": page_title, "http_status": page.get("status"), "elapsed_seconds": page.get("elapsed_seconds"), "link_count": len(seen_links), "form_count": min(len(parser.forms), DISCOVERY_MAX_FORMS), "button_count": min(len(seen_buttons), DISCOVERY_MAX_BUTTONS), "candidate_count": len(candidates), }, "candidates": candidates, } def _probe_site_url(host_entry: dict[str, str], port: int, scheme: str, timeout: float) -> dict[str, Any] | None: host = host_entry["host"] url = f"{scheme}://{host}:{port}/" try: with socket.create_connection((host, port), timeout=timeout): pass except OSError: return None started = datetime.now(timezone.utc) request = Request( url, headers={ "Accept": "text/html,application/xhtml+xml,application/json;q=0.8,*/*;q=0.5", "User-Agent": f"SystemTestController/{__version__} SiteScanner", }, ) try: with urlopen( request, timeout=timeout, context=UNVERIFIED_HTTPS_CONTEXT if scheme == "https" else None, ) as response: raw = response.read(256_000) headers = response.headers status = getattr(response, "status", 200) final_url = response.geturl() except HTTPError as exc: raw = exc.read(256_000) headers = exc.headers status = exc.code final_url = url except (URLError, TimeoutError, OSError, ssl.SSLError): return None elapsed = (datetime.now(timezone.utc) - started).total_seconds() content_type = headers.get("Content-Type", "") title = _extract_page_title(raw, content_type) server_name = headers.get("Server", "") name = title or server_name or f"{host_entry['ip']}:{port}" return { "name": name, "title": title, "url": url, "final_url": final_url, "ip": host_entry["ip"], "host": host, "port": port, "scheme": scheme, "status": status, "content_type": content_type, "elapsed_seconds": round(elapsed, 3), "address": f"{host_entry['ip']}:{port}", } def _probe_site(host_entry: dict[str, str], port: int, timeout: float) -> dict[str, Any] | None: for scheme in _scheme_candidates(port): result = _probe_site_url(host_entry, port, scheme, timeout) if result is not None: return result return None def run_preflight(config: ControllerConfig, db: Database) -> list[dict[str, Any]]: checks: list[dict[str, Any]] = [] db.init() database_health = db.health() checks.append( { "name": "database", "ok": database_health["ok"], "detail": database_health, } ) index_path = config.static_dir / "index.html" checks.append( { "name": "static_panel", "ok": index_path.exists(), "detail": {"path": str(index_path)}, } ) checks.append( { "name": "port", "ok": _port_is_available(config.host, config.port), "detail": {"host": config.host, "port": config.port}, } ) return checks def make_handler(config: ControllerConfig, db: Database): class ControllerRequestHandler(BaseHTTPRequestHandler): server_version = f"SystemTestController/{__version__}" def log_message(self, fmt: str, *args: Any) -> None: sys.stdout.write( "%s - - [%s] %s\n" % (self.client_address[0], self.log_date_time_string(), fmt % args) ) def _send_json(self, status: int, payload: Any) -> None: body = _json_bytes(payload) self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.end_headers() self.wfile.write(body) def _send_error_json(self, status: int, message: str, **extra: Any) -> None: payload = {"ok": False, "error": message} payload.update(extra) self._send_json(status, payload) def _read_json_body(self) -> dict[str, Any] | None: length = int(self.headers.get("Content-Length") or "0") if length <= 0: return {} raw_body = self.rfile.read(length) try: payload = json.loads(raw_body.decode("utf-8")) except json.JSONDecodeError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid JSON body") return None if not isinstance(payload, dict): self._send_error_json(HTTPStatus.BAD_REQUEST, "JSON body must be an object") return None return payload def _auth_agent(self, agent_id: int) -> dict[str, Any] | None: auth = self.headers.get("Authorization") or "" prefix = "Bearer " if not auth.startswith(prefix): self._send_error_json(HTTPStatus.UNAUTHORIZED, "Missing Bearer token") return None token = auth[len(prefix) :].strip() agent = db.validate_agent(agent_id, token) if agent is None: self._send_error_json(HTTPStatus.UNAUTHORIZED, "Invalid Bearer token") return None return agent def _health_payload(self) -> dict[str, Any]: db_health = db.health() panel_path = config.static_dir / "index.html" return { "ok": db_health["ok"] and panel_path.exists(), "service": "controller", "version": __version__, "name": config.controller_name, "deployment_mode": config.deployment_mode, "started_at": STARTED_AT, "public_base_url": config.public_base_url, "checks": { "database": db_health, "port": {"host": config.host, "port": config.port, "ok": True}, "static_panel": {"ok": panel_path.exists(), "path": str(panel_path)}, }, } def _discovery_payload(self) -> dict[str, Any]: host = self.headers.get("Host") base_url = f"http://{host}" if host else config.public_base_url return { "service": "Agent-S-Controller", "version": 1, "base_url": base_url.rstrip("/"), "time": datetime.now(timezone.utc).isoformat(timespec="seconds"), "deployment_mode": config.deployment_mode, } def do_OPTIONS(self) -> None: self.send_response(HTTPStatus.NO_CONTENT) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.end_headers() def do_GET(self) -> None: parsed = urlparse(self.path) path = parsed.path query = parse_qs(parsed.query) if path == "/health": self._send_json(HTTPStatus.OK, self._health_payload()) return if path == "/api/v1/discovery": self._send_json(HTTPStatus.OK, self._discovery_payload()) return if path == "/api/dashboard": self._send_json(HTTPStatus.OK, {"ok": True, **db.dashboard()}) return if path == "/api/agents": self._send_json(HTTPStatus.OK, {"ok": True, "agents": db.list_agents()}) return if path in {"/api/tasks", "/api/v1/tasks"}: self._send_json( HTTPStatus.OK, { "ok": True, "tasks": db.list_tasks( status=self._first_query(query, "status"), plan_name=self._first_query(query, "plan_name"), ), }, ) return if path == "/api/config/model": self._send_json(HTTPStatus.OK, {"ok": True, **db.get_model_config()}) return if path == "/api/tools": self._send_json(HTTPStatus.OK, {"ok": True, **db.get_tool_config()}) return if path == "/api/metrics": self._send_json(HTTPStatus.OK, {"ok": True, "metrics": db.metrics()}) return if path == "/api/reports/summary": self._send_json(HTTPStatus.OK, {"ok": True, "report": db.report_summary()}) return if path in {"/", "/index.html"}: self._send_static_file(config.static_dir / "index.html") return if path.startswith("/static/"): requested = path.removeprefix("/static/").replace("/", "\\") file_path = (config.static_dir / requested).resolve() try: file_path.relative_to(config.static_dir.resolve()) except ValueError: self._send_error_json(HTTPStatus.NOT_FOUND, "Static file not found") return self._send_static_file(file_path) return self._send_error_json(HTTPStatus.NOT_FOUND, "Route not found") def do_POST(self) -> None: parsed = urlparse(self.path) path = parsed.path payload = self._read_json_body() if payload is None: return if path == "/api/agents/register": self._handle_register(payload) return if path in {"/api/tasks", "/api/v1/tasks"}: created = db.create_task(payload) self._send_json(HTTPStatus.CREATED, {"ok": True, "task": created}) return if path == "/api/tasks/import": self._handle_import_tasks(payload) return if path == "/api/config/model": try: saved = db.save_model_config(payload) except ValueError as exc: self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._send_json(HTTPStatus.OK, {"ok": True, **saved}) return if path == "/api/models/test": self._send_json(HTTPStatus.OK, {"ok": True, "test": self._test_model(payload)}) return if path == "/api/sites/scan": try: result = self._scan_sites(payload) except ValueError as exc: self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._send_json(HTTPStatus.OK, {"ok": True, **result}) return if path == "/api/system-tests/start": try: result = self._start_system_test(payload) except ValueError as exc: self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._send_json(HTTPStatus.CREATED, {"ok": True, **result}) return if path == "/api/discovery/preview": try: result = self._preview_discovery(payload) except ValueError as exc: self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._send_json(HTTPStatus.OK, {"ok": True, **result}) return if path == "/api/discovery/import": try: result = self._import_discovery_tasks(payload) except ValueError as exc: self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._send_json(HTTPStatus.CREATED, {"ok": True, **result}) return if path == "/api/tools": saved = db.save_tool_config(payload) self._send_json(HTTPStatus.OK, {"ok": True, **saved}) return parts = [part for part in path.split("/") if part] if len(parts) == 4 and parts[:2] == ["api", "agents"] and parts[3] == "heartbeat": self._handle_heartbeat(parts[2], payload) return if len(parts) == 5 and parts[:2] == ["api", "agents"] and parts[3:] == ["tasks", "claim"]: self._handle_claim(parts[2]) return if len(parts) == 5 and parts[:2] == ["api", "agents"] and parts[3:] == ["model", "chat-json"]: self._handle_agent_model_chat(parts[2], payload) return if len(parts) == 6 and parts[:2] == ["api", "agents"] and parts[3] == "tasks" and parts[5] == "result": self._handle_agent_task_result(parts[2], parts[4], payload) return if len(parts) == 4 and parts[:2] == ["api", "tasks"]: self._handle_task_action(parts[2], parts[3], payload) return self._send_error_json(HTTPStatus.NOT_FOUND, "Route not found") def _first_query(self, query: dict[str, list[str]], name: str) -> str | None: values = query.get(name) or [] return values[0] if values else None def _handle_register(self, payload: dict[str, Any]) -> None: required = ["public_id", "machine_fingerprint"] missing = [key for key in required if not payload.get(key)] if missing: self._send_error_json( HTTPStatus.BAD_REQUEST, "Missing required registration fields", missing=missing, ) return try: registration = db.register_agent(payload) except AlreadyRegisteredError as exc: self._send_error_json( HTTPStatus.CONFLICT, str(exc) or "Agent public_id is already registered by another machine.", public_id=payload.get("public_id"), ) return status = HTTPStatus.CREATED if registration.get("registered") else HTTPStatus.OK self._send_json(status, {"ok": True, **registration}) def _handle_import_tasks(self, payload: dict[str, Any]) -> None: try: result = db.import_tasks(payload) except (ValueError, json.JSONDecodeError) as exc: self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc)) return self._send_json(HTTPStatus.CREATED, {"ok": True, **result}) def _handle_heartbeat(self, agent_id_text: str, payload: dict[str, Any]) -> None: try: agent_id = int(agent_id_text) except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid agent id") return if self._auth_agent(agent_id) is None: return receipt = db.record_heartbeat(agent_id, payload) self._send_json(HTTPStatus.OK, receipt) def _handle_claim(self, agent_id_text: str) -> None: try: agent_id = int(agent_id_text) except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid agent id") return if self._auth_agent(agent_id) is None: return task = db.claim_task(agent_id) self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) def _handle_agent_model_chat(self, agent_id_text: str, payload: dict[str, Any]) -> None: try: agent_id = int(agent_id_text) except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid agent id") return if self._auth_agent(agent_id) is None: return messages = payload.get("messages") if not isinstance(messages, list) or not messages: self._send_error_json(HTTPStatus.BAD_REQUEST, "Missing model messages") return if len(messages) > 20: self._send_error_json(HTTPStatus.BAD_REQUEST, "Too many model messages") return normalized_messages: list[dict[str, str]] = [] for message in messages: if not isinstance(message, dict): self._send_error_json(HTTPStatus.BAD_REQUEST, "Each model message must be an object") return role = str(message.get("role") or "").strip() content = str(message.get("content") or "") if role not in {"system", "user", "assistant"}: self._send_error_json(HTTPStatus.BAD_REQUEST, f"Invalid model message role: {role}") return normalized_messages.append({"role": role, "content": content}) result = self._proxy_model_chat_json( { "messages": normalized_messages, "temperature": payload.get("temperature"), } ) self._send_json(HTTPStatus.OK, result) def _handle_agent_task_result( self, agent_id_text: str, task_id_text: str, payload: dict[str, Any], ) -> None: try: agent_id = int(agent_id_text) task_id = int(task_id_text) except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid id") return if self._auth_agent(agent_id) is None: return task = db.record_task_result(task_id, payload) self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) def _handle_task_action( self, task_id_text: str, action: str, payload: dict[str, Any], ) -> None: try: task_id = int(task_id_text) except ValueError: self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid task id") return if action == "assign": agent_id = payload.get("agent_id") task = db.assign_task( task_id, int(agent_id) if agent_id not in {None, ""} else None, reason=payload.get("reason"), ) self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) return if action == "abort": task = db.update_task_status(task_id, "cancelled", reason=payload.get("reason") or "aborted by operator") self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) return if action == "retry": task = db.retry_task(task_id, reason=payload.get("reason")) self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) return if action == "status": status = str(payload.get("status") or "") if not status: self._send_error_json(HTTPStatus.BAD_REQUEST, "Missing status") return task = db.update_task_status(task_id, status, reason=payload.get("reason"), payload=payload.get("result")) self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) return if action == "result": task = db.record_task_result(task_id, payload) self._send_json(HTTPStatus.OK, {"ok": True, "task": task}) return self._send_error_json(HTTPStatus.NOT_FOUND, "Task action not found") def _scan_sites(self, payload: dict[str, Any]) -> dict[str, Any]: target, port_payload = _extract_site_scan_target(payload) ports = _parse_site_scan_ports(port_payload) timeout = float(payload.get("timeout_seconds") or 0.8) timeout = max(0.2, min(timeout, 5.0)) hosts = _site_scan_hosts(target) probe_count = len(hosts) * len(ports) if probe_count > MAX_SITE_SCAN_PROBES: raise ValueError("Scan range is too large; reduce hosts or ports and try again.") sites: list[dict[str, Any]] = [] worker_count = min(32, max(1, probe_count)) with ThreadPoolExecutor(max_workers=worker_count) as executor: futures = [ executor.submit(_probe_site, host_entry, port, timeout) for host_entry in hosts for port in ports ] for future in as_completed(futures): result = future.result() if result is not None: sites.append(result) sites.sort(key=lambda site: (site["ip"], site["port"], site["scheme"])) return { "target": target or "local", "hosts": hosts, "ports": ports, "scanned": probe_count, "count": len(sites), "sites": sites, } def _start_system_test(self, payload: dict[str, Any]) -> dict[str, Any]: stored = db.get_model_config(public=False)["value"] target_url = _normalize_discovery_url( payload.get("target_url") or payload.get("url") or stored.get("target_system_url") ) parsed = urlparse(target_url) display_name = str(payload.get("site_name") or parsed.netloc or target_url).strip() plan_name = str(payload.get("plan_name") or f"系统测试 - {display_name}").strip() if not plan_name: plan_name = "系统测试" try: max_depth = int(payload.get("max_depth") or payload.get("discovery_max_depth") or 2) except (TypeError, ValueError): max_depth = 2 max_depth = max(1, min(max_depth, 5)) assigned_agent_id: int | None = None raw_agent_id = payload.get("assigned_agent_id") or payload.get("agent_id") if raw_agent_id not in {None, ""}: try: assigned_agent_id = int(raw_agent_id) except (TypeError, ValueError) as exc: raise ValueError("Invalid controlled-agent id.") from exc root_id = str( payload.get("external_id") or _candidate_id(target_url, "system-test-root", datetime.now(timezone.utc).isoformat()) ) objective = ( "进入待测系统首页，观察页面可访问性、标题、链接、表单和主要入口；" "发现可拆分的功能后自动创建层级化子任务并加入主控端任务队列。" ) created = db.create_task( { "external_id": root_id, "title": f"启动系统测试：{display_name}", "plan_name": plan_name, "module": "系统入口", "parent_external_id": None, "root_external_id": root_id, "depth": 0, "task_level": "site", "risk_level": "low", "priority": 10, "failure_policy": "pause", "max_retries": 2, "timeout_seconds": int(payload.get("timeout_seconds") or 90), "assigned_agent_id": assigned_agent_id, "tags": ["system-test", "agent-discovery", "p0"], "payload": { "task_type": "smart_discovery_p0", "execution_mode": "web", "instruction": objective, "objective": objective, "target_url": target_url, "source": "system_test_start", "level": "site", "depth": 0, "risk_level": "low", "allow_child_discovery": True, "discovery_max_depth": max_depth, "open_in_browser": True, "capture_screenshot": True, "hierarchy": { "id": root_id, "parent_id": None, "root_id": root_id, "depth": 0, "level": "site", }, }, } ) try: db.save_model_config({**stored, "target_system_url": target_url}) except ValueError: pass active_agents = [ agent for agent in db.list_agents() if agent.get("status") in {"online", "idle", "running", "registered"} ] return { "task": created, "target_url": target_url, "plan_name": plan_name, "assigned_agent_id": assigned_agent_id, "active_agent_count": len(active_agents), "message": "System test root task created. A controlled agent will claim it from the queue.", } def _preview_discovery(self, payload: dict[str, Any]) -> dict[str, Any]: stored = db.get_model_config(public=False)["value"] target_url = _normalize_discovery_url( payload.get("target_url") or payload.get("url") or stored.get("target_system_url") ) timeout = float(payload.get("timeout_seconds") or stored.get("request_timeout_seconds") or 6) timeout = max(1.0, min(timeout, 20.0)) page = _fetch_discovery_page(target_url, timeout) return _build_discovery_candidates(target_url, page) def _import_discovery_tasks(self, payload: dict[str, Any]) -> dict[str, Any]: candidates = payload.get("tasks") or payload.get("candidates") or [] if not isinstance(candidates, list) or not candidates: raise ValueError("No discovery tasks were selected.") plan_name = str(payload.get("plan_name") or "智能探索 P0").strip() or "智能探索 P0" target_url = str(payload.get("target_url") or "").strip() created: list[dict[str, Any]] = [] id_map: dict[str, str] = {} selected_parent_ids = { str(item.get("parent_id")) for item in candidates if isinstance(item, dict) and item.get("parent_id") } for index, candidate in enumerate(candidates, start=1): if not isinstance(candidate, dict): continue source_id = str(candidate.get("id") or _candidate_id(target_url, "selected", index)) parent_id = candidate.get("parent_id") root_id = candidate.get("root_id") or source_id level = str(candidate.get("level") or "case") try: depth = int(candidate.get("depth") or 0) except (TypeError, ValueError): depth = 0 allow_child_discovery = ( source_id not in selected_parent_ids and level in {"site", "module"} and depth < 2 ) mapped_parent_id = id_map.get(str(parent_id), parent_id) if parent_id else None mapped_root_id = id_map.get(str(root_id), root_id) task_payload = { "task_type": "smart_discovery_p0", "instruction": candidate.get("objective") or candidate.get("title"), "objective": candidate.get("objective") or "", "target_url": candidate.get("target_url") or target_url, "source": candidate.get("source") or "discovery", "level": level, "depth": depth, "risk_level": candidate.get("risk_level") or "low", "allow_child_discovery": allow_child_discovery, "discovery_max_depth": 2, "preconditions": candidate.get("preconditions") or [], "discovery_candidate": candidate, "hierarchy": { "id": source_id, "parent_id": parent_id, "root_id": root_id, "depth": depth, "level": level, }, } if candidate.get("fields"): task_payload["fields"] = candidate.get("fields") if candidate.get("method"): task_payload["method"] = candidate.get("method") created_task = db.create_task( { "external_id": source_id, "title": candidate.get("title") or f"智能探索任务 {index}", "plan_name": plan_name, "module": candidate.get("module") or "", "parent_external_id": mapped_parent_id, "root_external_id": mapped_root_id, "depth": depth, "task_level": level, "risk_level": candidate.get("risk_level") or "low", "priority": candidate.get("priority") or 100, "failure_policy": candidate.get("failure_policy") or "skip", "max_retries": candidate.get("max_retries") or 2, "timeout_seconds": candidate.get("timeout_seconds") or 60, "tags": candidate.get("tags") or ["smart-discovery", "p0"], "payload": task_payload, } ) id_map[source_id] = str(created_task.get("external_id") or source_id) created.append(created_task) if not created: raise ValueError("No valid discovery tasks were selected.") return {"count": len(created), "tasks": created} def _proxy_model_chat_json(self, payload: dict[str, Any]) -> dict[str, Any]: stored = db.get_model_config(public=False)["value"] base_url = str(stored.get("vllm_base_url") or "").strip() model = str(stored.get("vllm_model") or "").strip() api_key = str(stored.get("api_key") or "") if not base_url or not model: return { "ok": False, "source": "controller_proxy", "error": "Model config is incomplete on controller.", "duration_seconds": 0, } try: timeout = float(stored.get("request_timeout_seconds") or 30) except (TypeError, ValueError): timeout = 30 timeout = max(1.0, min(timeout, 300.0)) temperature = payload.get("temperature") try: temperature_value = float(temperature) if temperature is not None else float(stored.get("temperature") or 0) except (TypeError, ValueError): temperature_value = 0 url = _chat_completions_url(base_url) request_payload = { "model": model, "messages": payload.get("messages") or [], "temperature": temperature_value, } body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8") headers = { "Accept": "application/json", "Content-Type": "application/json; charset=utf-8", } if api_key: headers["Authorization"] = f"Bearer {api_key}" request = Request(url, data=body, headers=headers, method="POST") started = datetime.now(timezone.utc) try: with urlopen(request, timeout=timeout) as response: raw = response.read(4_000_000) except HTTPError as exc: elapsed = (datetime.now(timezone.utc) - started).total_seconds() error_body = exc.read().decode("utf-8", errors="replace") return { "ok": False, "source": "controller_proxy", "url": url, "duration_seconds": round(elapsed, 3), "error": f"{url} -> HTTP {exc.code}: {error_body or exc.reason}", } except (URLError, TimeoutError, OSError) as exc: elapsed = (datetime.now(timezone.utc) - started).total_seconds() return { "ok": False, "source": "controller_proxy", "url": url, "duration_seconds": round(elapsed, 3), "error": f"{url} -> {exc}", } elapsed = (datetime.now(timezone.utc) - started).total_seconds() try: data = json.loads(raw.decode("utf-8")) if raw else {} except json.JSONDecodeError as exc: return { "ok": False, "source": "controller_proxy", "url": url, "duration_seconds": round(elapsed, 3), "error": f"Invalid model response: {exc}", } if not isinstance(data, dict): return { "ok": False, "source": "controller_proxy", "url": url, "duration_seconds": round(elapsed, 3), "error": "Invalid model response: root JSON is not an object.", } content = _completion_content(data) return { "ok": True, "source": "controller_proxy", "url": url, "duration_seconds": round(elapsed, 3), "content": content, "parsed": _parse_json_object(content), } def _test_model(self, payload: dict[str, Any]) -> dict[str, Any]: stored = db.get_model_config(public=False)["value"] base_url = str(payload.get("vllm_base_url") or stored.get("vllm_base_url") or "").rstrip("/") api_key = str(payload.get("api_key") or stored.get("api_key") or "") timeout = float(payload.get("request_timeout_seconds") or stored.get("request_timeout_seconds") or 5) if not base_url: return {"ok": False, "message": "Missing vLLM/OpenAI-compatible base URL"} url = base_url if url.endswith("/v1"): url = f"{url}/models" elif not url.endswith("/models"): url = f"{url}/v1/models" headers = {"Accept": "application/json"} if api_key: headers["Authorization"] = f"Bearer {api_key}" request = Request(url, headers=headers) started = datetime.now(timezone.utc) try: with urlopen(request, timeout=timeout) as response: raw = response.read(512_000) elapsed = (datetime.now(timezone.utc) - started).total_seconds() data = json.loads(raw.decode("utf-8")) if raw else {} models = data.get("data") if isinstance(data, dict) else None return { "ok": True, "url": url, "elapsed_seconds": round(elapsed, 3), "models": models[:20] if isinstance(models, list) else data, } except HTTPError as exc: return {"ok": False, "url": url, "message": f"HTTP {exc.code}: {exc.reason}"} except (URLError, TimeoutError, OSError, json.JSONDecodeError) as exc: return {"ok": False, "url": url, "message": str(exc)} def _send_static_file(self, file_path: Path) -> None: if not file_path.exists() or not file_path.is_file(): self._send_error_json(HTTPStatus.NOT_FOUND, "Static file not found") return body = file_path.read_bytes() content_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" self.send_response(HTTPStatus.OK) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) return ControllerRequestHandler def main() -> None: config = ControllerConfig.from_env() db = Database(config.database_path, config.agent_stale_after_seconds) checks = run_preflight(config, db) print("System Test Controller preflight:") for check in checks: status = "OK" if check["ok"] else "FAIL" print(f" [{status}] {check['name']}: {check['detail']}") failed = [check for check in checks if not check["ok"]] if failed: print("Controller did not start because preflight checks failed.", file=sys.stderr) raise SystemExit(2) handler = make_handler(config, db) server = ThreadingHTTPServer((config.host, config.port), handler) print(f"Controller started at {config.public_base_url}") print(f"Health: {config.public_base_url}/health") print(f"Panel: {config.public_base_url}/") if _browser_autostart_enabled(): print("Opening controller panel in browser...") _open_panel_later(config.public_base_url) try: server.serve_forever() except KeyboardInterrupt: print("\nController stopping...") finally: server.server_close()