1534 lines
61 KiB
Python
1534 lines
61 KiB
Python
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import html
|
||
import ipaddress
|
||
import mimetypes
|
||
import os
|
||
import re
|
||
import socket
|
||
import ssl
|
||
import sys
|
||
import threading
|
||
import webbrowser
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime, timezone
|
||
from http import HTTPStatus
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from urllib.error import HTTPError, URLError
|
||
from urllib.parse import parse_qs, urldefrag, urljoin, urlparse
|
||
from urllib.request import Request, urlopen
|
||
|
||
from . import __version__
|
||
from .config import ControllerConfig
|
||
from .storage import AlreadyRegisteredError, Database
|
||
|
||
|
||
STARTED_AT = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||
DEFAULT_SITE_SCAN_PORTS = [
|
||
80,
|
||
443,
|
||
3000,
|
||
4173,
|
||
5173,
|
||
5174,
|
||
8000,
|
||
8001,
|
||
8080,
|
||
8081,
|
||
8088,
|
||
8765,
|
||
9000,
|
||
9090,
|
||
10000,
|
||
]
|
||
TLS_SITE_PORTS = {443, 8443, 9443}
|
||
MAX_SITE_SCAN_PROBES = 512
|
||
MAX_SITE_SCAN_PORTS = 256
|
||
DISCOVERY_MAX_LINKS = 24
|
||
DISCOVERY_MAX_FORMS = 12
|
||
DISCOVERY_MAX_BUTTONS = 24
|
||
DISCOVERY_MAX_HTML_BYTES = 1_500_000
|
||
TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||
UNVERIFIED_HTTPS_CONTEXT = ssl._create_unverified_context()
|
||
|
||
|
||
def _browser_autostart_enabled() -> bool:
|
||
disabled_values = {"1", "true", "yes", "on"}
|
||
return (
|
||
os.environ.get("CONTROLLER_NO_BROWSER", "").strip().lower() not in disabled_values
|
||
and os.environ.get("AGENT_CONTROLLER_NO_BROWSER", "").strip().lower() not in disabled_values
|
||
)
|
||
|
||
|
||
def _open_panel_later(url: str) -> None:
|
||
def open_panel() -> None:
|
||
try:
|
||
webbrowser.open(url, new=2)
|
||
except Exception as exc: # Browser launch should not bring the service down.
|
||
print(f"Could not open browser automatically: {exc}", file=sys.stderr)
|
||
|
||
threading.Timer(0.8, open_panel).start()
|
||
|
||
|
||
def _json_bytes(payload: Any) -> bytes:
|
||
return json.dumps(payload, ensure_ascii=False, indent=2).encode("utf-8")
|
||
|
||
|
||
def _chat_completions_url(base_url: str) -> str:
|
||
base = base_url.rstrip("/")
|
||
if base.endswith("/chat/completions"):
|
||
return base
|
||
if base.endswith("/v1"):
|
||
return f"{base}/chat/completions"
|
||
return f"{base}/v1/chat/completions"
|
||
|
||
|
||
def _parse_json_object(text: str) -> dict[str, Any] | None:
|
||
raw = text.strip()
|
||
fence = re.search(r"```(?:json)?\s*(.*?)```", raw, re.IGNORECASE | re.DOTALL)
|
||
if fence:
|
||
raw = fence.group(1).strip()
|
||
start = raw.find("{")
|
||
end = raw.rfind("}")
|
||
if start >= 0 and end > start:
|
||
raw = raw[start : end + 1]
|
||
try:
|
||
parsed = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return None
|
||
return parsed if isinstance(parsed, dict) else None
|
||
|
||
|
||
def _completion_content(data: dict[str, Any]) -> str:
|
||
choices = data.get("choices")
|
||
if not isinstance(choices, list) or not choices:
|
||
return ""
|
||
choice = choices[0] if isinstance(choices[0], dict) else {}
|
||
message = choice.get("message") if isinstance(choice.get("message"), dict) else {}
|
||
content = (
|
||
message.get("content")
|
||
or message.get("reasoning_content")
|
||
or message.get("reasoning")
|
||
or choice.get("text")
|
||
or ""
|
||
)
|
||
if not isinstance(content, str):
|
||
return json.dumps(content, ensure_ascii=False)
|
||
return content
|
||
|
||
|
||
def _port_is_available(host: str, port: int) -> bool:
|
||
probe_host = "127.0.0.1" if host in {"0.0.0.0", "::", ""} else host
|
||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||
sock.settimeout(0.5)
|
||
return sock.connect_ex((probe_host, port)) != 0
|
||
|
||
|
||
def _is_blank(value: Any) -> bool:
|
||
return value is None or value == [] or (isinstance(value, str) and not value.strip())
|
||
|
||
|
||
def _parse_site_scan_ports(value: Any) -> list[int]:
|
||
if _is_blank(value):
|
||
return list(DEFAULT_SITE_SCAN_PORTS)
|
||
|
||
raw_parts = value if isinstance(value, list) else str(value).replace(";", ",").replace("，", ",").split(",")
|
||
ports: set[int] = set()
|
||
for raw_part in raw_parts:
|
||
text = str(raw_part).strip()
|
||
if not text:
|
||
continue
|
||
if "-" in text:
|
||
start_text, end_text = [part.strip() for part in text.split("-", 1)]
|
||
if not start_text.isdigit() or not end_text.isdigit():
|
||
raise ValueError(f"Invalid port range: {text}")
|
||
start, end = int(start_text), int(end_text)
|
||
if end < start:
|
||
raise ValueError(f"Invalid port range: {text}")
|
||
if end - start + 1 > MAX_SITE_SCAN_PORTS:
|
||
raise ValueError("Port range is too large; narrow it before scanning.")
|
||
candidates = range(start, end + 1)
|
||
else:
|
||
if not text.isdigit():
|
||
raise ValueError(f"Invalid port: {text}")
|
||
candidates = [int(text)]
|
||
for port in candidates:
|
||
if port < 1 or port > 65535:
|
||
raise ValueError(f"Port out of range: {port}")
|
||
ports.add(port)
|
||
|
||
if not ports:
|
||
raise ValueError("No valid ports to scan.")
|
||
if len(ports) > MAX_SITE_SCAN_PORTS:
|
||
raise ValueError("Too many ports to scan at once; please narrow the port list.")
|
||
return sorted(ports)
|
||
|
||
|
||
def _resolve_ipv4(host: str) -> str | None:
|
||
try:
|
||
infos = socket.getaddrinfo(host, None, socket.AF_INET, socket.SOCK_STREAM)
|
||
except socket.gaierror:
|
||
return None
|
||
for info in infos:
|
||
address = info[4][0]
|
||
if address:
|
||
return address
|
||
return None
|
||
|
||
|
||
def _add_local_ipv4(candidates: list[str], address: str | None) -> None:
|
||
if not address or address in candidates:
|
||
return
|
||
try:
|
||
parsed = ipaddress.ip_address(address)
|
||
except ValueError:
|
||
return
|
||
if parsed.version != 4 or parsed.is_unspecified or parsed.is_multicast:
|
||
return
|
||
candidates.append(address)
|
||
|
||
|
||
def _local_site_scan_hosts() -> list[dict[str, str]]:
|
||
addresses = ["127.0.0.1"]
|
||
try:
|
||
for address in socket.gethostbyname_ex(socket.gethostname())[2]:
|
||
_add_local_ipv4(addresses, address)
|
||
except OSError:
|
||
pass
|
||
try:
|
||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
|
||
sock.connect(("8.8.8.8", 80))
|
||
_add_local_ipv4(addresses, sock.getsockname()[0])
|
||
except OSError:
|
||
pass
|
||
return [{"host": address, "ip": address} for address in addresses]
|
||
|
||
|
||
def _site_scan_hosts(target: str) -> list[dict[str, str]]:
|
||
normalized = target.strip()
|
||
if normalized.lower() in {"", "local", "localhost", "127.0.0.1", "::1"}:
|
||
return _local_site_scan_hosts()
|
||
if any(marker in normalized for marker in ["/", "\\", ",", ";", " "]):
|
||
raise ValueError("Please enter one IP address or host name, not a network range.")
|
||
ip = _resolve_ipv4(normalized)
|
||
if ip is None:
|
||
raise ValueError(f"Unable to resolve host: {normalized}")
|
||
return [{"host": normalized, "ip": ip}]
|
||
|
||
|
||
def _extract_site_scan_target(payload: dict[str, Any]) -> tuple[str, Any]:
|
||
target = str(payload.get("host") or "").strip()
|
||
ports = payload.get("ports")
|
||
if "://" in target:
|
||
parsed = urlparse(target)
|
||
if parsed.hostname:
|
||
target = parsed.hostname
|
||
if _is_blank(ports) and parsed.port:
|
||
ports = parsed.port
|
||
elif target.count(":") == 1:
|
||
host_part, port_part = target.rsplit(":", 1)
|
||
if host_part.strip() and port_part.isdigit():
|
||
target = host_part.strip()
|
||
if _is_blank(ports):
|
||
ports = port_part
|
||
return target, ports
|
||
|
||
|
||
def _scheme_candidates(port: int) -> list[str]:
|
||
return ["https", "http"] if port in TLS_SITE_PORTS else ["http", "https"]
|
||
|
||
|
||
def _decode_body(raw: bytes, content_type: str) -> str:
|
||
charset = "utf-8"
|
||
match = re.search(r"charset=([^;\s]+)", content_type or "", re.IGNORECASE)
|
||
if match:
|
||
charset = match.group(1).strip("\"'")
|
||
for candidate in [charset, "utf-8", "gbk", "latin-1"]:
|
||
try:
|
||
return raw.decode(candidate, errors="replace")
|
||
except LookupError:
|
||
continue
|
||
return raw.decode("utf-8", errors="replace")
|
||
|
||
|
||
def _extract_page_title(raw: bytes, content_type: str) -> str:
|
||
text = _decode_body(raw, content_type)
|
||
match = TITLE_RE.search(text)
|
||
if not match:
|
||
return ""
|
||
return html.unescape(re.sub(r"\s+", " ", match.group(1))).strip()
|
||
|
||
|
||
def _clean_discovery_text(value: Any, max_length: int = 160) -> str:
|
||
text = html.unescape(re.sub(r"\s+", " ", str(value or ""))).strip()
|
||
return text[:max_length].strip()
|
||
|
||
|
||
def _normalize_discovery_url(value: Any) -> str:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
raise ValueError("Missing target system URL.")
|
||
if "://" not in text:
|
||
text = f"http://{text}"
|
||
parsed = urlparse(text)
|
||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||
raise ValueError("Target system URL must be an http(s) URL.")
|
||
return text
|
||
|
||
|
||
def _origin_tuple(value: str) -> tuple[str, str, int]:
|
||
parsed = urlparse(value)
|
||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||
return parsed.scheme, (parsed.hostname or "").lower(), port
|
||
|
||
|
||
def _same_origin(base_url: str, candidate_url: str) -> bool:
|
||
try:
|
||
return _origin_tuple(base_url) == _origin_tuple(candidate_url)
|
||
except ValueError:
|
||
return False
|
||
|
||
|
||
def _safe_join_page_url(base_url: str, href: str) -> str | None:
|
||
text = str(href or "").strip()
|
||
if not text or text.startswith(("#", "javascript:", "mailto:", "tel:", "data:")):
|
||
return None
|
||
joined = urljoin(base_url, text)
|
||
joined, _fragment = urldefrag(joined)
|
||
parsed = urlparse(joined)
|
||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||
return None
|
||
if not _same_origin(base_url, joined):
|
||
return None
|
||
return joined
|
||
|
||
|
||
class DiscoveryHtmlParser(HTMLParser):
|
||
def __init__(self) -> None:
|
||
super().__init__(convert_charrefs=True)
|
||
self.title_parts: list[str] = []
|
||
self.links: list[dict[str, Any]] = []
|
||
self.forms: list[dict[str, Any]] = []
|
||
self.buttons: list[dict[str, Any]] = []
|
||
self._in_title = False
|
||
self._anchor_stack: list[dict[str, Any]] = []
|
||
self._current_form: dict[str, Any] | None = None
|
||
self._current_button: dict[str, Any] | None = None
|
||
|
||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||
tag = tag.lower()
|
||
attr = {name.lower(): value or "" for name, value in attrs}
|
||
if tag == "title":
|
||
self._in_title = True
|
||
return
|
||
if tag == "a":
|
||
self._anchor_stack.append({"href": attr.get("href", ""), "text_parts": []})
|
||
return
|
||
if tag == "form":
|
||
self._current_form = {
|
||
"action": attr.get("action", ""),
|
||
"method": (attr.get("method") or "get").upper(),
|
||
"id": attr.get("id", ""),
|
||
"name": attr.get("name", ""),
|
||
"class": attr.get("class", ""),
|
||
"text_parts": [],
|
||
"fields": [],
|
||
}
|
||
return
|
||
if self._current_form is not None and tag in {"input", "select", "textarea"}:
|
||
self._current_form["fields"].append(
|
||
{
|
||
"tag": tag,
|
||
"type": (attr.get("type") or tag).lower(),
|
||
"name": attr.get("name", ""),
|
||
"id": attr.get("id", ""),
|
||
"placeholder": attr.get("placeholder", ""),
|
||
"aria_label": attr.get("aria-label", ""),
|
||
}
|
||
)
|
||
if tag == "button":
|
||
self._current_button = {
|
||
"type": attr.get("type", ""),
|
||
"id": attr.get("id", ""),
|
||
"name": attr.get("name", ""),
|
||
"class": attr.get("class", ""),
|
||
"text_parts": [],
|
||
"in_form": self._current_form is not None,
|
||
}
|
||
|
||
def handle_data(self, data: str) -> None:
|
||
if self._in_title:
|
||
self.title_parts.append(data)
|
||
if self._anchor_stack:
|
||
self._anchor_stack[-1]["text_parts"].append(data)
|
||
if self._current_form is not None:
|
||
self._current_form["text_parts"].append(data)
|
||
if self._current_button is not None:
|
||
self._current_button["text_parts"].append(data)
|
||
|
||
def handle_endtag(self, tag: str) -> None:
|
||
tag = tag.lower()
|
||
if tag == "title":
|
||
self._in_title = False
|
||
elif tag == "a" and self._anchor_stack:
|
||
anchor = self._anchor_stack.pop()
|
||
self.links.append(
|
||
{
|
||
"href": anchor.get("href", ""),
|
||
"text": _clean_discovery_text(" ".join(anchor.get("text_parts") or []), 120),
|
||
}
|
||
)
|
||
elif tag == "button" and self._current_button is not None:
|
||
button = self._current_button
|
||
button["text"] = _clean_discovery_text(" ".join(button.get("text_parts") or []), 120)
|
||
self.buttons.append(button)
|
||
self._current_button = None
|
||
elif tag == "form" and self._current_form is not None:
|
||
form = self._current_form
|
||
form["text"] = _clean_discovery_text(" ".join(form.get("text_parts") or []), 160)
|
||
self.forms.append(form)
|
||
self._current_form = None
|
||
|
||
|
||
def _fetch_discovery_page(url: str, timeout: float) -> dict[str, Any]:
|
||
request = Request(
|
||
url,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,*/*",
|
||
"User-Agent": f"SystemTestController/{__version__} discovery",
|
||
},
|
||
)
|
||
started = datetime.now(timezone.utc)
|
||
try:
|
||
with urlopen(request, timeout=timeout, context=UNVERIFIED_HTTPS_CONTEXT) as response:
|
||
raw = response.read(DISCOVERY_MAX_HTML_BYTES)
|
||
content_type = response.headers.get("Content-Type", "")
|
||
final_url = response.geturl()
|
||
status = response.status
|
||
except HTTPError as exc:
|
||
raw = exc.read(DISCOVERY_MAX_HTML_BYTES)
|
||
content_type = exc.headers.get("Content-Type", "")
|
||
final_url = exc.geturl()
|
||
status = exc.code
|
||
except (URLError, TimeoutError, OSError) as exc:
|
||
reason = getattr(exc, "reason", exc)
|
||
if isinstance(exc, TimeoutError) or isinstance(reason, TimeoutError):
|
||
reason = f"timed out after {timeout:g}s"
|
||
raise ValueError(f"Target system is not reachable: {reason}") from exc
|
||
|
||
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
||
return {
|
||
"url": url,
|
||
"final_url": final_url,
|
||
"status": status,
|
||
"content_type": content_type,
|
||
"html": _decode_body(raw, content_type),
|
||
"elapsed_seconds": round(elapsed, 3),
|
||
}
|
||
|
||
|
||
def _candidate_id(seed: str, *parts: Any) -> str:
|
||
raw = "|".join([seed, *[str(part or "") for part in parts]])
|
||
digest = hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12]
|
||
return f"auto-{digest}"
|
||
|
||
|
||
def _risk_for_text(text: str) -> str:
|
||
lowered = text.lower()
|
||
high = ["delete", "remove", "drop", "支付", "付款", "删除", "注销", "清空", "退款"]
|
||
medium = ["save", "submit", "create", "update", "edit", "upload", "保存", "提交", "新增", "修改", "上传"]
|
||
if any(word in lowered for word in high):
|
||
return "high"
|
||
if any(word in lowered for word in medium):
|
||
return "medium"
|
||
return "low"
|
||
|
||
|
||
def _looks_like_login_form(form: dict[str, Any]) -> bool:
|
||
fields = form.get("fields") or []
|
||
text = " ".join(
|
||
[
|
||
str(form.get("id") or ""),
|
||
str(form.get("name") or ""),
|
||
str(form.get("class") or ""),
|
||
str(form.get("text") or ""),
|
||
*[
|
||
" ".join(
|
||
[
|
||
str(field.get("type") or ""),
|
||
str(field.get("name") or ""),
|
||
str(field.get("id") or ""),
|
||
str(field.get("placeholder") or ""),
|
||
str(field.get("aria_label") or ""),
|
||
]
|
||
)
|
||
for field in fields
|
||
],
|
||
]
|
||
).lower()
|
||
return "password" in text or any(keyword in text for keyword in ["login", "signin", "sign in", "登录", "账号", "密码"])
|
||
|
||
|
||
def _build_discovery_candidates(target_url: str, page: dict[str, Any]) -> dict[str, Any]:
|
||
parser = DiscoveryHtmlParser()
|
||
try:
|
||
parser.feed(page.get("html") or "")
|
||
except Exception:
|
||
pass
|
||
|
||
page_url = page.get("final_url") or target_url
|
||
page_title = _clean_discovery_text(" ".join(parser.title_parts), 120) or _extract_page_title(
|
||
str(page.get("html") or "").encode("utf-8", errors="ignore"),
|
||
"text/html; charset=utf-8",
|
||
)
|
||
display_name = page_title or urlparse(page_url).netloc or page_url
|
||
root_id = _candidate_id(page_url, "root", display_name)
|
||
candidates: list[dict[str, Any]] = [
|
||
{
|
||
"id": root_id,
|
||
"parent_id": None,
|
||
"root_id": root_id,
|
||
"depth": 0,
|
||
"level": "site",
|
||
"title": f"站点冒烟检查：{display_name}",
|
||
"objective": "确认待测站点可以访问，并记录页面标题、状态码和基础元信息。",
|
||
"target_url": page_url,
|
||
"module": "站点入口",
|
||
"risk_level": "low",
|
||
"priority": 20,
|
||
"max_retries": 2,
|
||
"failure_policy": "pause",
|
||
"preconditions": [],
|
||
"source": "page",
|
||
"tags": ["smart-discovery", "p0", "site"],
|
||
"selected": True,
|
||
}
|
||
]
|
||
|
||
seen_links: set[str] = set()
|
||
for link in parser.links:
|
||
if len(seen_links) >= DISCOVERY_MAX_LINKS:
|
||
break
|
||
href = _safe_join_page_url(page_url, str(link.get("href") or ""))
|
||
if not href or href in seen_links:
|
||
continue
|
||
seen_links.add(href)
|
||
label = _clean_discovery_text(link.get("text"), 80) or urlparse(href).path.strip("/") or href
|
||
risk = _risk_for_text(label)
|
||
candidate_id = _candidate_id(page_url, "link", href, label)
|
||
candidates.append(
|
||
{
|
||
"id": candidate_id,
|
||
"parent_id": root_id,
|
||
"root_id": root_id,
|
||
"depth": 1,
|
||
"level": "module",
|
||
"title": f"探索页面：{label}",
|
||
"objective": f"访问并检查页面“{label}”是否可达，记录页面标题和返回状态。",
|
||
"target_url": href,
|
||
"module": label,
|
||
"risk_level": risk,
|
||
"priority": 40 if risk == "low" else 70,
|
||
"max_retries": 2,
|
||
"failure_policy": "pause" if risk == "high" else "skip",
|
||
"preconditions": [],
|
||
"source": "html_link",
|
||
"tags": ["smart-discovery", "p0", "navigation"],
|
||
"selected": risk != "high",
|
||
}
|
||
)
|
||
|
||
for index, form in enumerate(parser.forms[:DISCOVERY_MAX_FORMS], start=1):
|
||
action_url = _safe_join_page_url(page_url, str(form.get("action") or "")) or page_url
|
||
is_login = _looks_like_login_form(form)
|
||
field_names = [
|
||
_clean_discovery_text(
|
||
field.get("placeholder") or field.get("aria_label") or field.get("name") or field.get("id") or field.get("type"),
|
||
40,
|
||
)
|
||
for field in form.get("fields") or []
|
||
]
|
||
field_names = [name for name in field_names if name][:8]
|
||
label = "登录表单" if is_login else (_clean_discovery_text(form.get("name") or form.get("id") or form.get("text"), 60) or f"表单 {index}")
|
||
risk = "medium" if is_login else _risk_for_text(label)
|
||
candidate_id = _candidate_id(page_url, "form", index, action_url, label)
|
||
preconditions = ["需要测试账号/密码"] if is_login else []
|
||
candidates.append(
|
||
{
|
||
"id": candidate_id,
|
||
"parent_id": root_id,
|
||
"root_id": root_id,
|
||
"depth": 1,
|
||
"level": "feature",
|
||
"title": f"识别功能：{label}",
|
||
"objective": f"分析{label}的输入项与提交入口，P0 阶段先确认页面元素存在，不执行真实提交。",
|
||
"target_url": action_url,
|
||
"module": label,
|
||
"risk_level": risk,
|
||
"priority": 35 if is_login else 55,
|
||
"max_retries": 2,
|
||
"failure_policy": "pause" if is_login else "skip",
|
||
"preconditions": preconditions,
|
||
"source": "html_form",
|
||
"method": form.get("method") or "GET",
|
||
"fields": field_names,
|
||
"tags": ["smart-discovery", "p0", "form"],
|
||
"selected": True,
|
||
}
|
||
)
|
||
|
||
seen_buttons: set[str] = set()
|
||
for button in parser.buttons:
|
||
if len(seen_buttons) >= DISCOVERY_MAX_BUTTONS:
|
||
break
|
||
label = _clean_discovery_text(button.get("text") or button.get("name") or button.get("id"), 60)
|
||
if not label or label in seen_buttons:
|
||
continue
|
||
seen_buttons.add(label)
|
||
risk = _risk_for_text(label)
|
||
candidate_id = _candidate_id(page_url, "button", label)
|
||
candidates.append(
|
||
{
|
||
"id": candidate_id,
|
||
"parent_id": root_id,
|
||
"root_id": root_id,
|
||
"depth": 1,
|
||
"level": "feature",
|
||
"title": f"识别入口：{label}",
|
||
"objective": f"确认入口“{label}”在页面中存在；高风险入口仅记录，不自动执行破坏性动作。",
|
||
"target_url": page_url,
|
||
"module": "页面入口",
|
||
"risk_level": risk,
|
||
"priority": 60 if risk == "low" else 85,
|
||
"max_retries": 1,
|
||
"failure_policy": "pause" if risk == "high" else "skip",
|
||
"preconditions": [],
|
||
"source": "html_button",
|
||
"tags": ["smart-discovery", "p0", "entry"],
|
||
"selected": risk != "high",
|
||
}
|
||
)
|
||
|
||
if len(candidates) == 1:
|
||
for label, objective in [
|
||
("单页应用基础冒烟测试", "确认单页应用根页面可达，并为后续 GUI 探索提供初始页面。"),
|
||
("页面导航与菜单发现", "后续由 GUI Agent 在页面内识别菜单、按钮和可交互区域。"),
|
||
]:
|
||
candidate_id = _candidate_id(page_url, "fallback", label)
|
||
candidates.append(
|
||
{
|
||
"id": candidate_id,
|
||
"parent_id": root_id,
|
||
"root_id": root_id,
|
||
"depth": 1,
|
||
"level": "module",
|
||
"title": label,
|
||
"objective": objective,
|
||
"target_url": page_url,
|
||
"module": "页面探索",
|
||
"risk_level": "low",
|
||
"priority": 50,
|
||
"max_retries": 2,
|
||
"failure_policy": "skip",
|
||
"preconditions": [],
|
||
"source": "fallback",
|
||
"tags": ["smart-discovery", "p0", "spa"],
|
||
"selected": True,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"summary": {
|
||
"target_url": target_url,
|
||
"final_url": page_url,
|
||
"page_title": page_title,
|
||
"http_status": page.get("status"),
|
||
"elapsed_seconds": page.get("elapsed_seconds"),
|
||
"link_count": len(seen_links),
|
||
"form_count": min(len(parser.forms), DISCOVERY_MAX_FORMS),
|
||
"button_count": min(len(seen_buttons), DISCOVERY_MAX_BUTTONS),
|
||
"candidate_count": len(candidates),
|
||
},
|
||
"candidates": candidates,
|
||
}
|
||
|
||
|
||
def _probe_site_url(host_entry: dict[str, str], port: int, scheme: str, timeout: float) -> dict[str, Any] | None:
|
||
host = host_entry["host"]
|
||
url = f"{scheme}://{host}:{port}/"
|
||
try:
|
||
with socket.create_connection((host, port), timeout=timeout):
|
||
pass
|
||
except OSError:
|
||
return None
|
||
|
||
started = datetime.now(timezone.utc)
|
||
request = Request(
|
||
url,
|
||
headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/json;q=0.8,*/*;q=0.5",
|
||
"User-Agent": f"SystemTestController/{__version__} SiteScanner",
|
||
},
|
||
)
|
||
try:
|
||
with urlopen(
|
||
request,
|
||
timeout=timeout,
|
||
context=UNVERIFIED_HTTPS_CONTEXT if scheme == "https" else None,
|
||
) as response:
|
||
raw = response.read(256_000)
|
||
headers = response.headers
|
||
status = getattr(response, "status", 200)
|
||
final_url = response.geturl()
|
||
except HTTPError as exc:
|
||
raw = exc.read(256_000)
|
||
headers = exc.headers
|
||
status = exc.code
|
||
final_url = url
|
||
except (URLError, TimeoutError, OSError, ssl.SSLError):
|
||
return None
|
||
|
||
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
||
content_type = headers.get("Content-Type", "")
|
||
title = _extract_page_title(raw, content_type)
|
||
server_name = headers.get("Server", "")
|
||
name = title or server_name or f"{host_entry['ip']}:{port}"
|
||
return {
|
||
"name": name,
|
||
"title": title,
|
||
"url": url,
|
||
"final_url": final_url,
|
||
"ip": host_entry["ip"],
|
||
"host": host,
|
||
"port": port,
|
||
"scheme": scheme,
|
||
"status": status,
|
||
"content_type": content_type,
|
||
"elapsed_seconds": round(elapsed, 3),
|
||
"address": f"{host_entry['ip']}:{port}",
|
||
}
|
||
|
||
|
||
def _probe_site(host_entry: dict[str, str], port: int, timeout: float) -> dict[str, Any] | None:
|
||
for scheme in _scheme_candidates(port):
|
||
result = _probe_site_url(host_entry, port, scheme, timeout)
|
||
if result is not None:
|
||
return result
|
||
return None
|
||
|
||
|
||
def run_preflight(config: ControllerConfig, db: Database) -> list[dict[str, Any]]:
|
||
checks: list[dict[str, Any]] = []
|
||
|
||
db.init()
|
||
database_health = db.health()
|
||
checks.append(
|
||
{
|
||
"name": "database",
|
||
"ok": database_health["ok"],
|
||
"detail": database_health,
|
||
}
|
||
)
|
||
|
||
index_path = config.static_dir / "index.html"
|
||
checks.append(
|
||
{
|
||
"name": "static_panel",
|
||
"ok": index_path.exists(),
|
||
"detail": {"path": str(index_path)},
|
||
}
|
||
)
|
||
|
||
checks.append(
|
||
{
|
||
"name": "port",
|
||
"ok": _port_is_available(config.host, config.port),
|
||
"detail": {"host": config.host, "port": config.port},
|
||
}
|
||
)
|
||
|
||
return checks
|
||
|
||
|
||
def make_handler(config: ControllerConfig, db: Database):
|
||
class ControllerRequestHandler(BaseHTTPRequestHandler):
|
||
server_version = f"SystemTestController/{__version__}"
|
||
|
||
def log_message(self, fmt: str, *args: Any) -> None:
|
||
sys.stdout.write(
|
||
"%s - - [%s] %s\n"
|
||
% (self.client_address[0], self.log_date_time_string(), fmt % args)
|
||
)
|
||
|
||
def _send_json(self, status: int, payload: Any) -> None:
|
||
body = _json_bytes(payload)
|
||
self.send_response(status)
|
||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.send_header("Access-Control-Allow-Origin", "*")
|
||
self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization")
|
||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||
self.end_headers()
|
||
self.wfile.write(body)
|
||
|
||
def _send_error_json(self, status: int, message: str, **extra: Any) -> None:
|
||
payload = {"ok": False, "error": message}
|
||
payload.update(extra)
|
||
self._send_json(status, payload)
|
||
|
||
def _read_json_body(self) -> dict[str, Any] | None:
|
||
length = int(self.headers.get("Content-Length") or "0")
|
||
if length <= 0:
|
||
return {}
|
||
raw_body = self.rfile.read(length)
|
||
try:
|
||
payload = json.loads(raw_body.decode("utf-8"))
|
||
except json.JSONDecodeError:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid JSON body")
|
||
return None
|
||
if not isinstance(payload, dict):
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "JSON body must be an object")
|
||
return None
|
||
return payload
|
||
|
||
def _auth_agent(self, agent_id: int) -> dict[str, Any] | None:
|
||
auth = self.headers.get("Authorization") or ""
|
||
prefix = "Bearer "
|
||
if not auth.startswith(prefix):
|
||
self._send_error_json(HTTPStatus.UNAUTHORIZED, "Missing Bearer token")
|
||
return None
|
||
token = auth[len(prefix) :].strip()
|
||
agent = db.validate_agent(agent_id, token)
|
||
if agent is None:
|
||
self._send_error_json(HTTPStatus.UNAUTHORIZED, "Invalid Bearer token")
|
||
return None
|
||
return agent
|
||
|
||
def _health_payload(self) -> dict[str, Any]:
|
||
db_health = db.health()
|
||
panel_path = config.static_dir / "index.html"
|
||
return {
|
||
"ok": db_health["ok"] and panel_path.exists(),
|
||
"service": "controller",
|
||
"version": __version__,
|
||
"name": config.controller_name,
|
||
"deployment_mode": config.deployment_mode,
|
||
"started_at": STARTED_AT,
|
||
"public_base_url": config.public_base_url,
|
||
"checks": {
|
||
"database": db_health,
|
||
"port": {"host": config.host, "port": config.port, "ok": True},
|
||
"static_panel": {"ok": panel_path.exists(), "path": str(panel_path)},
|
||
},
|
||
}
|
||
|
||
def _discovery_payload(self) -> dict[str, Any]:
|
||
host = self.headers.get("Host")
|
||
base_url = f"http://{host}" if host else config.public_base_url
|
||
return {
|
||
"service": "Agent-S-Controller",
|
||
"version": 1,
|
||
"base_url": base_url.rstrip("/"),
|
||
"time": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||
"deployment_mode": config.deployment_mode,
|
||
}
|
||
|
||
def do_OPTIONS(self) -> None:
|
||
self.send_response(HTTPStatus.NO_CONTENT)
|
||
self.send_header("Access-Control-Allow-Origin", "*")
|
||
self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization")
|
||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||
self.end_headers()
|
||
|
||
def do_GET(self) -> None:
|
||
parsed = urlparse(self.path)
|
||
path = parsed.path
|
||
query = parse_qs(parsed.query)
|
||
if path == "/health":
|
||
self._send_json(HTTPStatus.OK, self._health_payload())
|
||
return
|
||
if path == "/api/v1/discovery":
|
||
self._send_json(HTTPStatus.OK, self._discovery_payload())
|
||
return
|
||
if path == "/api/dashboard":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **db.dashboard()})
|
||
return
|
||
if path == "/api/agents":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "agents": db.list_agents()})
|
||
return
|
||
if path in {"/api/tasks", "/api/v1/tasks"}:
|
||
self._send_json(
|
||
HTTPStatus.OK,
|
||
{
|
||
"ok": True,
|
||
"tasks": db.list_tasks(
|
||
status=self._first_query(query, "status"),
|
||
plan_name=self._first_query(query, "plan_name"),
|
||
),
|
||
},
|
||
)
|
||
return
|
||
if path == "/api/config/model":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **db.get_model_config()})
|
||
return
|
||
if path == "/api/tools":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **db.get_tool_config()})
|
||
return
|
||
if path == "/api/metrics":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "metrics": db.metrics()})
|
||
return
|
||
if path == "/api/reports/summary":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "report": db.report_summary()})
|
||
return
|
||
if path in {"/", "/index.html"}:
|
||
self._send_static_file(config.static_dir / "index.html")
|
||
return
|
||
if path.startswith("/static/"):
|
||
requested = path.removeprefix("/static/").replace("/", "\\")
|
||
file_path = (config.static_dir / requested).resolve()
|
||
try:
|
||
file_path.relative_to(config.static_dir.resolve())
|
||
except ValueError:
|
||
self._send_error_json(HTTPStatus.NOT_FOUND, "Static file not found")
|
||
return
|
||
self._send_static_file(file_path)
|
||
return
|
||
self._send_error_json(HTTPStatus.NOT_FOUND, "Route not found")
|
||
|
||
def do_POST(self) -> None:
|
||
parsed = urlparse(self.path)
|
||
path = parsed.path
|
||
payload = self._read_json_body()
|
||
if payload is None:
|
||
return
|
||
|
||
if path == "/api/agents/register":
|
||
self._handle_register(payload)
|
||
return
|
||
if path in {"/api/tasks", "/api/v1/tasks"}:
|
||
created = db.create_task(payload)
|
||
self._send_json(HTTPStatus.CREATED, {"ok": True, "task": created})
|
||
return
|
||
if path == "/api/tasks/import":
|
||
self._handle_import_tasks(payload)
|
||
return
|
||
if path == "/api/config/model":
|
||
try:
|
||
saved = db.save_model_config(payload)
|
||
except ValueError as exc:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||
return
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **saved})
|
||
return
|
||
if path == "/api/models/test":
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "test": self._test_model(payload)})
|
||
return
|
||
if path == "/api/sites/scan":
|
||
try:
|
||
result = self._scan_sites(payload)
|
||
except ValueError as exc:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||
return
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **result})
|
||
return
|
||
if path == "/api/system-tests/start":
|
||
try:
|
||
result = self._start_system_test(payload)
|
||
except ValueError as exc:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||
return
|
||
self._send_json(HTTPStatus.CREATED, {"ok": True, **result})
|
||
return
|
||
if path == "/api/discovery/preview":
|
||
try:
|
||
result = self._preview_discovery(payload)
|
||
except ValueError as exc:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||
return
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **result})
|
||
return
|
||
if path == "/api/discovery/import":
|
||
try:
|
||
result = self._import_discovery_tasks(payload)
|
||
except ValueError as exc:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||
return
|
||
self._send_json(HTTPStatus.CREATED, {"ok": True, **result})
|
||
return
|
||
if path == "/api/tools":
|
||
saved = db.save_tool_config(payload)
|
||
self._send_json(HTTPStatus.OK, {"ok": True, **saved})
|
||
return
|
||
|
||
parts = [part for part in path.split("/") if part]
|
||
if len(parts) == 4 and parts[:2] == ["api", "agents"] and parts[3] == "heartbeat":
|
||
self._handle_heartbeat(parts[2], payload)
|
||
return
|
||
if len(parts) == 5 and parts[:2] == ["api", "agents"] and parts[3:] == ["tasks", "claim"]:
|
||
self._handle_claim(parts[2])
|
||
return
|
||
if len(parts) == 5 and parts[:2] == ["api", "agents"] and parts[3:] == ["model", "chat-json"]:
|
||
self._handle_agent_model_chat(parts[2], payload)
|
||
return
|
||
if len(parts) == 6 and parts[:2] == ["api", "agents"] and parts[3] == "tasks" and parts[5] == "result":
|
||
self._handle_agent_task_result(parts[2], parts[4], payload)
|
||
return
|
||
if len(parts) == 4 and parts[:2] == ["api", "tasks"]:
|
||
self._handle_task_action(parts[2], parts[3], payload)
|
||
return
|
||
|
||
self._send_error_json(HTTPStatus.NOT_FOUND, "Route not found")
|
||
|
||
def _first_query(self, query: dict[str, list[str]], name: str) -> str | None:
|
||
values = query.get(name) or []
|
||
return values[0] if values else None
|
||
|
||
def _handle_register(self, payload: dict[str, Any]) -> None:
|
||
required = ["public_id", "machine_fingerprint"]
|
||
missing = [key for key in required if not payload.get(key)]
|
||
if missing:
|
||
self._send_error_json(
|
||
HTTPStatus.BAD_REQUEST,
|
||
"Missing required registration fields",
|
||
missing=missing,
|
||
)
|
||
return
|
||
try:
|
||
registration = db.register_agent(payload)
|
||
except AlreadyRegisteredError as exc:
|
||
self._send_error_json(
|
||
HTTPStatus.CONFLICT,
|
||
str(exc)
|
||
or "Agent public_id is already registered by another machine.",
|
||
public_id=payload.get("public_id"),
|
||
)
|
||
return
|
||
status = HTTPStatus.CREATED if registration.get("registered") else HTTPStatus.OK
|
||
self._send_json(status, {"ok": True, **registration})
|
||
|
||
def _handle_import_tasks(self, payload: dict[str, Any]) -> None:
|
||
try:
|
||
result = db.import_tasks(payload)
|
||
except (ValueError, json.JSONDecodeError) as exc:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, str(exc))
|
||
return
|
||
self._send_json(HTTPStatus.CREATED, {"ok": True, **result})
|
||
|
||
def _handle_heartbeat(self, agent_id_text: str, payload: dict[str, Any]) -> None:
|
||
try:
|
||
agent_id = int(agent_id_text)
|
||
except ValueError:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid agent id")
|
||
return
|
||
if self._auth_agent(agent_id) is None:
|
||
return
|
||
receipt = db.record_heartbeat(agent_id, payload)
|
||
self._send_json(HTTPStatus.OK, receipt)
|
||
|
||
def _handle_claim(self, agent_id_text: str) -> None:
|
||
try:
|
||
agent_id = int(agent_id_text)
|
||
except ValueError:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid agent id")
|
||
return
|
||
if self._auth_agent(agent_id) is None:
|
||
return
|
||
task = db.claim_task(agent_id)
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
|
||
def _handle_agent_model_chat(self, agent_id_text: str, payload: dict[str, Any]) -> None:
|
||
try:
|
||
agent_id = int(agent_id_text)
|
||
except ValueError:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid agent id")
|
||
return
|
||
if self._auth_agent(agent_id) is None:
|
||
return
|
||
messages = payload.get("messages")
|
||
if not isinstance(messages, list) or not messages:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Missing model messages")
|
||
return
|
||
if len(messages) > 20:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Too many model messages")
|
||
return
|
||
normalized_messages: list[dict[str, str]] = []
|
||
for message in messages:
|
||
if not isinstance(message, dict):
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Each model message must be an object")
|
||
return
|
||
role = str(message.get("role") or "").strip()
|
||
content = str(message.get("content") or "")
|
||
if role not in {"system", "user", "assistant"}:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, f"Invalid model message role: {role}")
|
||
return
|
||
normalized_messages.append({"role": role, "content": content})
|
||
result = self._proxy_model_chat_json(
|
||
{
|
||
"messages": normalized_messages,
|
||
"temperature": payload.get("temperature"),
|
||
}
|
||
)
|
||
self._send_json(HTTPStatus.OK, result)
|
||
|
||
def _handle_agent_task_result(
|
||
self,
|
||
agent_id_text: str,
|
||
task_id_text: str,
|
||
payload: dict[str, Any],
|
||
) -> None:
|
||
try:
|
||
agent_id = int(agent_id_text)
|
||
task_id = int(task_id_text)
|
||
except ValueError:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid id")
|
||
return
|
||
if self._auth_agent(agent_id) is None:
|
||
return
|
||
task = db.record_task_result(task_id, payload)
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
|
||
def _handle_task_action(
|
||
self,
|
||
task_id_text: str,
|
||
action: str,
|
||
payload: dict[str, Any],
|
||
) -> None:
|
||
try:
|
||
task_id = int(task_id_text)
|
||
except ValueError:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Invalid task id")
|
||
return
|
||
|
||
if action == "assign":
|
||
agent_id = payload.get("agent_id")
|
||
task = db.assign_task(
|
||
task_id,
|
||
int(agent_id) if agent_id not in {None, ""} else None,
|
||
reason=payload.get("reason"),
|
||
)
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
return
|
||
if action == "abort":
|
||
task = db.update_task_status(task_id, "cancelled", reason=payload.get("reason") or "aborted by operator")
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
return
|
||
if action == "retry":
|
||
task = db.retry_task(task_id, reason=payload.get("reason"))
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
return
|
||
if action == "status":
|
||
status = str(payload.get("status") or "")
|
||
if not status:
|
||
self._send_error_json(HTTPStatus.BAD_REQUEST, "Missing status")
|
||
return
|
||
task = db.update_task_status(task_id, status, reason=payload.get("reason"), payload=payload.get("result"))
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
return
|
||
if action == "result":
|
||
task = db.record_task_result(task_id, payload)
|
||
self._send_json(HTTPStatus.OK, {"ok": True, "task": task})
|
||
return
|
||
self._send_error_json(HTTPStatus.NOT_FOUND, "Task action not found")
|
||
|
||
def _scan_sites(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
target, port_payload = _extract_site_scan_target(payload)
|
||
ports = _parse_site_scan_ports(port_payload)
|
||
timeout = float(payload.get("timeout_seconds") or 0.8)
|
||
timeout = max(0.2, min(timeout, 5.0))
|
||
hosts = _site_scan_hosts(target)
|
||
probe_count = len(hosts) * len(ports)
|
||
if probe_count > MAX_SITE_SCAN_PROBES:
|
||
raise ValueError("Scan range is too large; reduce hosts or ports and try again.")
|
||
|
||
sites: list[dict[str, Any]] = []
|
||
worker_count = min(32, max(1, probe_count))
|
||
with ThreadPoolExecutor(max_workers=worker_count) as executor:
|
||
futures = [
|
||
executor.submit(_probe_site, host_entry, port, timeout)
|
||
for host_entry in hosts
|
||
for port in ports
|
||
]
|
||
for future in as_completed(futures):
|
||
result = future.result()
|
||
if result is not None:
|
||
sites.append(result)
|
||
|
||
sites.sort(key=lambda site: (site["ip"], site["port"], site["scheme"]))
|
||
return {
|
||
"target": target or "local",
|
||
"hosts": hosts,
|
||
"ports": ports,
|
||
"scanned": probe_count,
|
||
"count": len(sites),
|
||
"sites": sites,
|
||
}
|
||
|
||
def _start_system_test(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
stored = db.get_model_config(public=False)["value"]
|
||
target_url = _normalize_discovery_url(
|
||
payload.get("target_url")
|
||
or payload.get("url")
|
||
or stored.get("target_system_url")
|
||
)
|
||
parsed = urlparse(target_url)
|
||
display_name = str(payload.get("site_name") or parsed.netloc or target_url).strip()
|
||
plan_name = str(payload.get("plan_name") or f"系统测试 - {display_name}").strip()
|
||
if not plan_name:
|
||
plan_name = "系统测试"
|
||
|
||
try:
|
||
max_depth = int(payload.get("max_depth") or payload.get("discovery_max_depth") or 2)
|
||
except (TypeError, ValueError):
|
||
max_depth = 2
|
||
max_depth = max(1, min(max_depth, 5))
|
||
|
||
assigned_agent_id: int | None = None
|
||
raw_agent_id = payload.get("assigned_agent_id") or payload.get("agent_id")
|
||
if raw_agent_id not in {None, ""}:
|
||
try:
|
||
assigned_agent_id = int(raw_agent_id)
|
||
except (TypeError, ValueError) as exc:
|
||
raise ValueError("Invalid controlled-agent id.") from exc
|
||
|
||
root_id = str(
|
||
payload.get("external_id")
|
||
or _candidate_id(target_url, "system-test-root", datetime.now(timezone.utc).isoformat())
|
||
)
|
||
objective = (
|
||
"进入待测系统首页，观察页面可访问性、标题、链接、表单和主要入口；"
|
||
"发现可拆分的功能后自动创建层级化子任务并加入主控端任务队列。"
|
||
)
|
||
created = db.create_task(
|
||
{
|
||
"external_id": root_id,
|
||
"title": f"启动系统测试：{display_name}",
|
||
"plan_name": plan_name,
|
||
"module": "系统入口",
|
||
"parent_external_id": None,
|
||
"root_external_id": root_id,
|
||
"depth": 0,
|
||
"task_level": "site",
|
||
"risk_level": "low",
|
||
"priority": 10,
|
||
"failure_policy": "pause",
|
||
"max_retries": 2,
|
||
"timeout_seconds": int(payload.get("timeout_seconds") or 90),
|
||
"assigned_agent_id": assigned_agent_id,
|
||
"tags": ["system-test", "agent-discovery", "p0"],
|
||
"payload": {
|
||
"task_type": "smart_discovery_p0",
|
||
"execution_mode": "web",
|
||
"instruction": objective,
|
||
"objective": objective,
|
||
"target_url": target_url,
|
||
"source": "system_test_start",
|
||
"level": "site",
|
||
"depth": 0,
|
||
"risk_level": "low",
|
||
"allow_child_discovery": True,
|
||
"discovery_max_depth": max_depth,
|
||
"open_in_browser": True,
|
||
"capture_screenshot": True,
|
||
"hierarchy": {
|
||
"id": root_id,
|
||
"parent_id": None,
|
||
"root_id": root_id,
|
||
"depth": 0,
|
||
"level": "site",
|
||
},
|
||
},
|
||
}
|
||
)
|
||
|
||
try:
|
||
db.save_model_config({**stored, "target_system_url": target_url})
|
||
except ValueError:
|
||
pass
|
||
|
||
active_agents = [
|
||
agent
|
||
for agent in db.list_agents()
|
||
if agent.get("status") in {"online", "idle", "running", "registered"}
|
||
]
|
||
return {
|
||
"task": created,
|
||
"target_url": target_url,
|
||
"plan_name": plan_name,
|
||
"assigned_agent_id": assigned_agent_id,
|
||
"active_agent_count": len(active_agents),
|
||
"message": "System test root task created. A controlled agent will claim it from the queue.",
|
||
}
|
||
|
||
def _preview_discovery(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
stored = db.get_model_config(public=False)["value"]
|
||
target_url = _normalize_discovery_url(
|
||
payload.get("target_url")
|
||
or payload.get("url")
|
||
or stored.get("target_system_url")
|
||
)
|
||
timeout = float(payload.get("timeout_seconds") or stored.get("request_timeout_seconds") or 6)
|
||
timeout = max(1.0, min(timeout, 20.0))
|
||
page = _fetch_discovery_page(target_url, timeout)
|
||
return _build_discovery_candidates(target_url, page)
|
||
|
||
def _import_discovery_tasks(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
candidates = payload.get("tasks") or payload.get("candidates") or []
|
||
if not isinstance(candidates, list) or not candidates:
|
||
raise ValueError("No discovery tasks were selected.")
|
||
|
||
plan_name = str(payload.get("plan_name") or "智能探索 P0").strip() or "智能探索 P0"
|
||
target_url = str(payload.get("target_url") or "").strip()
|
||
created: list[dict[str, Any]] = []
|
||
id_map: dict[str, str] = {}
|
||
selected_parent_ids = {
|
||
str(item.get("parent_id"))
|
||
for item in candidates
|
||
if isinstance(item, dict) and item.get("parent_id")
|
||
}
|
||
for index, candidate in enumerate(candidates, start=1):
|
||
if not isinstance(candidate, dict):
|
||
continue
|
||
source_id = str(candidate.get("id") or _candidate_id(target_url, "selected", index))
|
||
parent_id = candidate.get("parent_id")
|
||
root_id = candidate.get("root_id") or source_id
|
||
level = str(candidate.get("level") or "case")
|
||
try:
|
||
depth = int(candidate.get("depth") or 0)
|
||
except (TypeError, ValueError):
|
||
depth = 0
|
||
allow_child_discovery = (
|
||
source_id not in selected_parent_ids
|
||
and level in {"site", "module"}
|
||
and depth < 2
|
||
)
|
||
mapped_parent_id = id_map.get(str(parent_id), parent_id) if parent_id else None
|
||
mapped_root_id = id_map.get(str(root_id), root_id)
|
||
task_payload = {
|
||
"task_type": "smart_discovery_p0",
|
||
"instruction": candidate.get("objective") or candidate.get("title"),
|
||
"objective": candidate.get("objective") or "",
|
||
"target_url": candidate.get("target_url") or target_url,
|
||
"source": candidate.get("source") or "discovery",
|
||
"level": level,
|
||
"depth": depth,
|
||
"risk_level": candidate.get("risk_level") or "low",
|
||
"allow_child_discovery": allow_child_discovery,
|
||
"discovery_max_depth": 2,
|
||
"preconditions": candidate.get("preconditions") or [],
|
||
"discovery_candidate": candidate,
|
||
"hierarchy": {
|
||
"id": source_id,
|
||
"parent_id": parent_id,
|
||
"root_id": root_id,
|
||
"depth": depth,
|
||
"level": level,
|
||
},
|
||
}
|
||
if candidate.get("fields"):
|
||
task_payload["fields"] = candidate.get("fields")
|
||
if candidate.get("method"):
|
||
task_payload["method"] = candidate.get("method")
|
||
|
||
created_task = db.create_task(
|
||
{
|
||
"external_id": source_id,
|
||
"title": candidate.get("title") or f"智能探索任务 {index}",
|
||
"plan_name": plan_name,
|
||
"module": candidate.get("module") or "",
|
||
"parent_external_id": mapped_parent_id,
|
||
"root_external_id": mapped_root_id,
|
||
"depth": depth,
|
||
"task_level": level,
|
||
"risk_level": candidate.get("risk_level") or "low",
|
||
"priority": candidate.get("priority") or 100,
|
||
"failure_policy": candidate.get("failure_policy") or "skip",
|
||
"max_retries": candidate.get("max_retries") or 2,
|
||
"timeout_seconds": candidate.get("timeout_seconds") or 60,
|
||
"tags": candidate.get("tags") or ["smart-discovery", "p0"],
|
||
"payload": task_payload,
|
||
}
|
||
)
|
||
id_map[source_id] = str(created_task.get("external_id") or source_id)
|
||
created.append(created_task)
|
||
|
||
if not created:
|
||
raise ValueError("No valid discovery tasks were selected.")
|
||
return {"count": len(created), "tasks": created}
|
||
|
||
def _proxy_model_chat_json(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
stored = db.get_model_config(public=False)["value"]
|
||
base_url = str(stored.get("vllm_base_url") or "").strip()
|
||
model = str(stored.get("vllm_model") or "").strip()
|
||
api_key = str(stored.get("api_key") or "")
|
||
if not base_url or not model:
|
||
return {
|
||
"ok": False,
|
||
"source": "controller_proxy",
|
||
"error": "Model config is incomplete on controller.",
|
||
"duration_seconds": 0,
|
||
}
|
||
|
||
try:
|
||
timeout = float(stored.get("request_timeout_seconds") or 30)
|
||
except (TypeError, ValueError):
|
||
timeout = 30
|
||
timeout = max(1.0, min(timeout, 300.0))
|
||
temperature = payload.get("temperature")
|
||
try:
|
||
temperature_value = float(temperature) if temperature is not None else float(stored.get("temperature") or 0)
|
||
except (TypeError, ValueError):
|
||
temperature_value = 0
|
||
|
||
url = _chat_completions_url(base_url)
|
||
request_payload = {
|
||
"model": model,
|
||
"messages": payload.get("messages") or [],
|
||
"temperature": temperature_value,
|
||
}
|
||
body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8")
|
||
headers = {
|
||
"Accept": "application/json",
|
||
"Content-Type": "application/json; charset=utf-8",
|
||
}
|
||
if api_key:
|
||
headers["Authorization"] = f"Bearer {api_key}"
|
||
|
||
request = Request(url, data=body, headers=headers, method="POST")
|
||
started = datetime.now(timezone.utc)
|
||
try:
|
||
with urlopen(request, timeout=timeout) as response:
|
||
raw = response.read(4_000_000)
|
||
except HTTPError as exc:
|
||
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
||
error_body = exc.read().decode("utf-8", errors="replace")
|
||
return {
|
||
"ok": False,
|
||
"source": "controller_proxy",
|
||
"url": url,
|
||
"duration_seconds": round(elapsed, 3),
|
||
"error": f"{url} -> HTTP {exc.code}: {error_body or exc.reason}",
|
||
}
|
||
except (URLError, TimeoutError, OSError) as exc:
|
||
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
||
return {
|
||
"ok": False,
|
||
"source": "controller_proxy",
|
||
"url": url,
|
||
"duration_seconds": round(elapsed, 3),
|
||
"error": f"{url} -> {exc}",
|
||
}
|
||
|
||
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
||
try:
|
||
data = json.loads(raw.decode("utf-8")) if raw else {}
|
||
except json.JSONDecodeError as exc:
|
||
return {
|
||
"ok": False,
|
||
"source": "controller_proxy",
|
||
"url": url,
|
||
"duration_seconds": round(elapsed, 3),
|
||
"error": f"Invalid model response: {exc}",
|
||
}
|
||
if not isinstance(data, dict):
|
||
return {
|
||
"ok": False,
|
||
"source": "controller_proxy",
|
||
"url": url,
|
||
"duration_seconds": round(elapsed, 3),
|
||
"error": "Invalid model response: root JSON is not an object.",
|
||
}
|
||
|
||
content = _completion_content(data)
|
||
return {
|
||
"ok": True,
|
||
"source": "controller_proxy",
|
||
"url": url,
|
||
"duration_seconds": round(elapsed, 3),
|
||
"content": content,
|
||
"parsed": _parse_json_object(content),
|
||
}
|
||
|
||
def _test_model(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
stored = db.get_model_config(public=False)["value"]
|
||
base_url = str(payload.get("vllm_base_url") or stored.get("vllm_base_url") or "").rstrip("/")
|
||
api_key = str(payload.get("api_key") or stored.get("api_key") or "")
|
||
timeout = float(payload.get("request_timeout_seconds") or stored.get("request_timeout_seconds") or 5)
|
||
if not base_url:
|
||
return {"ok": False, "message": "Missing vLLM/OpenAI-compatible base URL"}
|
||
url = base_url
|
||
if url.endswith("/v1"):
|
||
url = f"{url}/models"
|
||
elif not url.endswith("/models"):
|
||
url = f"{url}/v1/models"
|
||
headers = {"Accept": "application/json"}
|
||
if api_key:
|
||
headers["Authorization"] = f"Bearer {api_key}"
|
||
request = Request(url, headers=headers)
|
||
started = datetime.now(timezone.utc)
|
||
try:
|
||
with urlopen(request, timeout=timeout) as response:
|
||
raw = response.read(512_000)
|
||
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
|
||
data = json.loads(raw.decode("utf-8")) if raw else {}
|
||
models = data.get("data") if isinstance(data, dict) else None
|
||
return {
|
||
"ok": True,
|
||
"url": url,
|
||
"elapsed_seconds": round(elapsed, 3),
|
||
"models": models[:20] if isinstance(models, list) else data,
|
||
}
|
||
except HTTPError as exc:
|
||
return {"ok": False, "url": url, "message": f"HTTP {exc.code}: {exc.reason}"}
|
||
except (URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||
return {"ok": False, "url": url, "message": str(exc)}
|
||
|
||
def _send_static_file(self, file_path: Path) -> None:
|
||
if not file_path.exists() or not file_path.is_file():
|
||
self._send_error_json(HTTPStatus.NOT_FOUND, "Static file not found")
|
||
return
|
||
body = file_path.read_bytes()
|
||
content_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
|
||
self.send_response(HTTPStatus.OK)
|
||
self.send_header("Content-Type", content_type)
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
self.wfile.write(body)
|
||
|
||
return ControllerRequestHandler
|
||
|
||
|
||
def main() -> None:
|
||
config = ControllerConfig.from_env()
|
||
db = Database(config.database_path, config.agent_stale_after_seconds)
|
||
checks = run_preflight(config, db)
|
||
|
||
print("System Test Controller preflight:")
|
||
for check in checks:
|
||
status = "OK" if check["ok"] else "FAIL"
|
||
print(f" [{status}] {check['name']}: {check['detail']}")
|
||
|
||
failed = [check for check in checks if not check["ok"]]
|
||
if failed:
|
||
print("Controller did not start because preflight checks failed.", file=sys.stderr)
|
||
raise SystemExit(2)
|
||
|
||
handler = make_handler(config, db)
|
||
server = ThreadingHTTPServer((config.host, config.port), handler)
|
||
print(f"Controller started at {config.public_base_url}")
|
||
print(f"Health: {config.public_base_url}/health")
|
||
print(f"Panel: {config.public_base_url}/")
|
||
if _browser_autostart_enabled():
|
||
print("Opening controller panel in browser...")
|
||
_open_panel_later(config.public_base_url)
|
||
try:
|
||
server.serve_forever()
|
||
except KeyboardInterrupt:
|
||
print("\nController stopping...")
|
||
finally:
|
||
server.server_close()
|