实现terminalcode和文件交互
This commit is contained in:
minorQAQ committed
1 parent
287e1935f6
commit
a5e5aff63f
141 files changed
+2385
-1826
No files matched your search
+289
-372
@@ -1,389 +1,306 @@
|
||||
# import os
|
||||
# import re
|
||||
# import json
|
||||
# import argparse
|
||||
# from pathlib import Path
|
||||
# from typing import Dict, Any, List, Optional
|
||||
# from urllib.parse import urljoin, urlparse
|
||||
|
||||
# from selenium import webdriver
|
||||
# from selenium.webdriver.edge.options import Options
|
||||
# from selenium.webdriver.edge.service import Service
|
||||
# from selenium.webdriver.support.ui import WebDriverWait
|
||||
# from webdriver_manager.microsoft import EdgeChromiumDriverManager
|
||||
|
||||
|
||||
# # -------------------- 工具函数 --------------------
|
||||
# def get_absolute_url(base_url: str, href: str) -> Optional[str]:
|
||||
# if not href:
|
||||
# return None
|
||||
# href = href.strip()
|
||||
# if href.startswith(('javascript:', 'mailto:', 'tel:', '#')):
|
||||
# return None
|
||||
# absolute = urljoin(base_url, href)
|
||||
# parsed = urlparse(absolute)
|
||||
# if parsed.scheme in ('http', 'https', 'file'):
|
||||
# return absolute
|
||||
# return None
|
||||
|
||||
|
||||
# # -------------------- 语义上下文推断 --------------------
|
||||
# def infer_context_from_attrs(tag_name: str, attrs: Dict[str, str], text: str = "") -> str:
|
||||
# class_list = attrs.get("class", "").split()
|
||||
# id_ = attrs.get("id", "")
|
||||
# href = attrs.get("href", "")
|
||||
# features = " ".join(class_list) + " " + id_
|
||||
|
||||
# if tag_name == "a":
|
||||
# if any(w in features for w in ["btn", "button", "tab", "nav-btn", "menu-item", "action-btn"]):
|
||||
# return "按钮"
|
||||
# if "cm.bilibili.com" in href or "ad" in features:
|
||||
# return "广告链接"
|
||||
# return "普通链接"
|
||||
|
||||
# if "search" in features or "nav-search" in features:
|
||||
# return "搜索框"
|
||||
# if any(w in features for w in ["header", "top-bar", "masthead", "nav"]):
|
||||
# return "顶部导航"
|
||||
# if any(w in features for w in ["sidebar", "aside", "right-entry"]):
|
||||
# return "侧边栏"
|
||||
# if "footer" in features:
|
||||
# return "页脚"
|
||||
# if any(w in features for w in ["channel", "partition", "channel-icons"]):
|
||||
# return "频道/分区"
|
||||
# if any(w in features for w in ["carousel", "banner", "slider"]):
|
||||
# return "轮播/横幅"
|
||||
# if any(w in features for w in ["video-card", "feed-card", "card-item", "card"]):
|
||||
# return "视频卡片"
|
||||
# if "main" in features or "content" in features:
|
||||
# return "主要内容区"
|
||||
# if tag_name in ("input", "textarea"):
|
||||
# return "输入控件"
|
||||
# if tag_name == "button":
|
||||
# return "按钮"
|
||||
# if tag_name == "select":
|
||||
# return "下拉选择"
|
||||
# if tag_name in ("div", "li", "span", "img", "td", "label"):
|
||||
# if any(w in features for w in ["btn", "button", "action", "clickable", "tab", "menu-item"]):
|
||||
# return "按钮"
|
||||
# if any(w in features for w in ["card", "item", "cell", "list-item"]):
|
||||
# return "卡片/列表项"
|
||||
# if "dropdown" in features or "menu" in features:
|
||||
# return "菜单"
|
||||
# if "dialog" in features or "modal" in features:
|
||||
# return "弹窗"
|
||||
# if "icon" in features:
|
||||
# return "图标"
|
||||
# if tag_name == "img":
|
||||
# return "图片"
|
||||
# return "交互区域"
|
||||
# return "其他"
|
||||
|
||||
|
||||
# # -------------------- 控件提取（去重 + 仅保留元素大小） --------------------
|
||||
# def extract_controls(driver) -> List[Dict[str, Any]]:
|
||||
# """
|
||||
# 无头模式下提取所有可交互元素（去重），
|
||||
# 仅返回元素的尺寸（CSS 像素），不包含坐标。
|
||||
# """
|
||||
# raw = driver.execute_script("""
|
||||
# const tags = ['a', 'input', 'button', 'select', 'textarea',
|
||||
# 'div', 'li', 'span', 'img', 'td', 'label'];
|
||||
# let allElements = [];
|
||||
|
||||
# // 1. 收集候选元素
|
||||
# for (let tag of tags) {
|
||||
# let elems = document.querySelectorAll(tag);
|
||||
# for (let elem of elems) {
|
||||
# if (tag === 'input' && elem.type === 'hidden') continue;
|
||||
# if (tag === 'a' && !elem.href && !elem.onclick) continue;
|
||||
|
||||
# let isInteractive = false;
|
||||
# if (['div', 'li', 'span', 'img', 'td', 'label'].includes(tag)) {
|
||||
# if (elem.onclick) isInteractive = true;
|
||||
# let role = elem.getAttribute('role');
|
||||
# if (role && ['button', 'link', 'tab', 'menuitem'].includes(role)) isInteractive = true;
|
||||
# if (elem.hasAttribute('tabindex') && elem.getAttribute('tabindex') !== '-1') isInteractive = true;
|
||||
# if (elem.hasAttribute('data-action') || elem.hasAttribute('data-href')) isInteractive = true;
|
||||
# let cls = elem.className || '';
|
||||
# if (/(btn|button|clickable|action|link|tab|menu-item|card|cell)/i.test(cls)) {
|
||||
# isInteractive = true;
|
||||
# }
|
||||
# if (!isInteractive) continue;
|
||||
# }
|
||||
# allElements.push(elem);
|
||||
# }
|
||||
# }
|
||||
|
||||
# // 2. 去重：只保留不被其他任何元素包含的最外层元素
|
||||
# let uniqueElements = [];
|
||||
# for (let elem of allElements) {
|
||||
# let isChild = false;
|
||||
# for (let other of allElements) {
|
||||
# if (other !== elem && other.contains(elem)) {
|
||||
# isChild = true;
|
||||
# break;
|
||||
# }
|
||||
# }
|
||||
# if (!isChild) {
|
||||
# uniqueElements.push(elem);
|
||||
# }
|
||||
# }
|
||||
|
||||
# // 3. 获取每个元素的属性、文本和尺寸
|
||||
# return uniqueElements.map(elem => {
|
||||
# let tag = elem.tagName.toLowerCase();
|
||||
# let attrs = {};
|
||||
# for (let attr of elem.attributes) {
|
||||
# attrs[attr.name] = attr.value;
|
||||
# }
|
||||
# let formAction = null;
|
||||
# if (tag === 'button' || (tag === 'input' && ['submit','button','image'].includes(elem.type))) {
|
||||
# let form = elem.closest('form');
|
||||
# if (form) formAction = form.getAttribute('action') || '';
|
||||
# }
|
||||
# let text = elem.innerText ? elem.innerText.trim().substring(0, 50) : '';
|
||||
# let rect = elem.getBoundingClientRect();
|
||||
|
||||
# return {
|
||||
# tag: tag,
|
||||
# attributes: attrs,
|
||||
# text: text,
|
||||
# form_action: formAction,
|
||||
# is_interactive: true,
|
||||
# size: {
|
||||
# width: rect.width,
|
||||
# height: rect.height
|
||||
# }
|
||||
# };
|
||||
# });
|
||||
# """)
|
||||
|
||||
# controls = []
|
||||
# for item in raw:
|
||||
# ctrl = {
|
||||
# "type": item["tag"],
|
||||
# "attributes": item["attributes"],
|
||||
# "text": item.get("text", ""),
|
||||
# "context": infer_context_from_attrs(item["tag"], item["attributes"], item.get("text", "")),
|
||||
# "form_action": item.get("form_action"),
|
||||
# "is_interactive": item.get("is_interactive", False),
|
||||
# "size": item.get("size", {})
|
||||
# }
|
||||
# controls.append(ctrl)
|
||||
# return controls
|
||||
|
||||
|
||||
# # -------------------- 人性化报告生成 --------------------
|
||||
# def short_text_for_control(ctrl: Dict[str, Any]) -> str:
|
||||
# tp = ctrl["type"]
|
||||
# attrs = ctrl["attributes"]
|
||||
# is_interactive = ctrl.get("is_interactive", False)
|
||||
# size = ctrl.get("size", {})
|
||||
# size_str = ""
|
||||
# if size:
|
||||
# w, h = size.get("width", 0), size.get("height", 0)
|
||||
# size_str = f" [大小:{w:.0f}x{h:.0f}]"
|
||||
|
||||
# base = ""
|
||||
# if tp == "a":
|
||||
# href = attrs.get("href", "")
|
||||
# text = ctrl.get("text") or attrs.get("title") or attrs.get("aria-label") or href[:60]
|
||||
# base = f"链接 →{text}"
|
||||
# if href:
|
||||
# base += f" : {href}"
|
||||
# elif tp == "button":
|
||||
# text = ctrl.get("text") or attrs.get("value") or attrs.get("aria-label") or "按钮"
|
||||
# base = f"按钮: {text}"
|
||||
# elif tp == "input":
|
||||
# input_type = attrs.get("type", "text")
|
||||
# text = ctrl.get("text") or attrs.get("value") or attrs.get("placeholder") or ""
|
||||
# base = f"输入框 ({input_type})"
|
||||
# if text:
|
||||
# base += f" [{text}]"
|
||||
# elif tp == "textarea":
|
||||
# text = ctrl.get("text") or attrs.get("placeholder") or ""
|
||||
# base = "多行文本框"
|
||||
# if text:
|
||||
# base += f" [{text}]"
|
||||
# elif tp == "select":
|
||||
# text = ctrl.get("text") or ""
|
||||
# base = "下拉选择框"
|
||||
# if text:
|
||||
# base += f" [{text}]"
|
||||
# elif is_interactive:
|
||||
# text = ctrl.get("text") or ""
|
||||
# role = attrs.get("role", "")
|
||||
# tag_desc = f"{tp}控件"
|
||||
# if role == "button":
|
||||
# tag_desc = "按钮(ARIA)"
|
||||
# elif role == "link":
|
||||
# tag_desc = "链接(ARIA)"
|
||||
# elif "card" in (attrs.get("class", "") + attrs.get("id", "")):
|
||||
# tag_desc = "卡片"
|
||||
# base = f"可交互{tag_desc}"
|
||||
# if text:
|
||||
# base += f" [{text}]"
|
||||
# if attrs.get("data-href"):
|
||||
# base += f" (目标: {attrs['data-href']})"
|
||||
# else:
|
||||
# text = ctrl.get("text") or ""
|
||||
# base = f"{tp} 控件"
|
||||
# if text:
|
||||
# base += f" [{text}]"
|
||||
# return base + size_str
|
||||
|
||||
|
||||
# def generate_report(url: str, controls: List[Dict[str, Any]]) -> str:
|
||||
# lines = [f"🌐 页面交互控件分析报告（静态单页·无头模式）\n页面: {url}\n" + "=" * 50]
|
||||
# groups: Dict[str, List[Dict]] = {}
|
||||
# for ctrl in controls:
|
||||
# ctx = ctrl["context"]
|
||||
# groups.setdefault(ctx, []).append(ctrl)
|
||||
|
||||
# sorted_groups = sorted(groups.items(), key=lambda x: x[0])
|
||||
# for context, items in sorted_groups:
|
||||
# lines.append(f"├── 【{context}】({len(items)}个)")
|
||||
# for item in items:
|
||||
# desc = short_text_for_control(item)
|
||||
# lines.append(f"│ ├── {desc}")
|
||||
# return "\n".join(lines)
|
||||
|
||||
|
||||
# # -------------------- 主程序 --------------------
|
||||
# def main():
|
||||
# parser = argparse.ArgumentParser(description="静态单页交互控件分析（Edge·无头模式·仅元素大小）")
|
||||
# parser.add_argument("source", help="起始 URL 或本地 HTML 文件路径")
|
||||
# parser.add_argument("-o", "--output", help="输出报告文件路径，默认打印到控制台")
|
||||
# parser.add_argument("--json", action="store_true", help="同时保存机器可读的 JSON 文件（与 -o 同名 .json）")
|
||||
# args = parser.parse_args()
|
||||
|
||||
# # 构造 URL
|
||||
# if args.source.startswith(('http://', 'https://')):
|
||||
# url = args.source
|
||||
# else:
|
||||
# abs_path = os.path.abspath(args.source)
|
||||
# url = Path(abs_path).as_uri()
|
||||
|
||||
# # 无头模式 Edge，设置窗口大小
|
||||
# options = Options()
|
||||
# options.add_argument("--headless=new") # 新版无头模式
|
||||
# options.add_argument("--window-size=1920,1080") # 固定视口大小
|
||||
# options.add_argument("--disable-gpu") # 禁用 GPU 加速（无头下建议）
|
||||
|
||||
# service = Service(EdgeChromiumDriverManager().install())
|
||||
# driver = webdriver.Edge(service=service, options=options)
|
||||
|
||||
# try:
|
||||
# driver.get(url)
|
||||
# WebDriverWait(driver, 15).until(
|
||||
# lambda d: d.execute_script("return document.readyState") == "complete"
|
||||
# )
|
||||
# controls = extract_controls(driver)
|
||||
# finally:
|
||||
# driver.quit()
|
||||
|
||||
# report = generate_report(url, controls)
|
||||
|
||||
# # 输出结果
|
||||
# if args.output:
|
||||
# with open(args.output, 'w', encoding='utf-8') as f:
|
||||
# f.write(report)
|
||||
# print(f"人性化报告已保存到 {args.output}")
|
||||
# if args.json:
|
||||
# json_path = args.output.rsplit('.', 1)[0] + '.json'
|
||||
# with open(json_path, 'w', encoding='utf-8') as jf:
|
||||
# json.dump({
|
||||
# "url": url,
|
||||
# "controls": controls
|
||||
# }, jf, ensure_ascii=False, indent=2)
|
||||
# print(f"机器可读 JSON 已保存到 {json_path}")
|
||||
# else:
|
||||
# print(report)
|
||||
|
||||
|
||||
# if __name__ == "__main__":
|
||||
# main()
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import requests
|
||||
from PIL import Image
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
# ================= 配置 =================
|
||||
IMAGE_PATH = r"C:\Users\86166\Desktop\3.png"
|
||||
RESIZED_PATH = r"C:\Users\86166\Desktop\3_resized.png"
|
||||
OUTPUT_FILE = r"C:\Users\86166\Desktop\model_output.txt"
|
||||
API_URL = "http://localhost:4555/v1/chat/completions"
|
||||
MODEL_NAME = "UI-TARS-1.5-7B"
|
||||
TARGET_SIZE = (1000, 1000) # 宽, 高
|
||||
INSTRUCTION = "点击屏幕中央。请只输出点击的x和y坐标，格式为'x,y'，不要输出任何其他内容。"
|
||||
# =======================================
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.edge.options import Options
|
||||
from selenium.webdriver.edge.service import Service
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from webdriver_manager.microsoft import EdgeChromiumDriverManager
|
||||
|
||||
def resize_image_force(image_path, save_path, size):
|
||||
"""强制拉伸图片到指定尺寸（不保持比例）"""
|
||||
img = Image.open(image_path).convert("RGB")
|
||||
img_resized = img.resize(size, Image.Resampling.LANCZOS)
|
||||
img_resized.save(save_path, "PNG")
|
||||
print(f"图片已缩放至 {size[0]}×{size[1]}，保存至: {save_path}")
|
||||
return save_path
|
||||
|
||||
def image_to_base64(image_path):
|
||||
"""将图片转为 base64 字符串（去除换行）"""
|
||||
with open(image_path, "rb") as f:
|
||||
img_bytes = f.read()
|
||||
b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
return b64
|
||||
# -------------------- 工具函数 --------------------
|
||||
def get_absolute_url(base_url: str, href: str) -> Optional[str]:
|
||||
if not href:
|
||||
return None
|
||||
href = href.strip()
|
||||
if href.startswith(('javascript:', 'mailto:', 'tel:', '#')):
|
||||
return None
|
||||
absolute = urljoin(base_url, href)
|
||||
parsed = urlparse(absolute)
|
||||
if parsed.scheme in ('http', 'https', 'file'):
|
||||
return absolute
|
||||
return None
|
||||
|
||||
def call_model(image_path, instruction):
|
||||
"""调用 vLLM 服务"""
|
||||
b64 = image_to_base64(image_path)
|
||||
data_url = f"data:image/png;base64,{b64}"
|
||||
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": instruction}
|
||||
]
|
||||
# -------------------- 语义上下文推断 --------------------
|
||||
def infer_context_from_attrs(tag_name: str, attrs: Dict[str, str], text: str = "") -> str:
|
||||
class_list = attrs.get("class", "").split()
|
||||
id_ = attrs.get("id", "")
|
||||
href = attrs.get("href", "")
|
||||
features = " ".join(class_list) + " " + id_
|
||||
|
||||
if tag_name == "a":
|
||||
if any(w in features for w in ["btn", "button", "tab", "nav-btn", "menu-item", "action-btn"]):
|
||||
return "按钮"
|
||||
if "cm.bilibili.com" in href or "ad" in features:
|
||||
return "广告链接"
|
||||
return "普通链接"
|
||||
|
||||
if "search" in features or "nav-search" in features:
|
||||
return "搜索框"
|
||||
if any(w in features for w in ["header", "top-bar", "masthead", "nav"]):
|
||||
return "顶部导航"
|
||||
if any(w in features for w in ["sidebar", "aside", "right-entry"]):
|
||||
return "侧边栏"
|
||||
if "footer" in features:
|
||||
return "页脚"
|
||||
if any(w in features for w in ["channel", "partition", "channel-icons"]):
|
||||
return "频道/分区"
|
||||
if any(w in features for w in ["carousel", "banner", "slider"]):
|
||||
return "轮播/横幅"
|
||||
if any(w in features for w in ["video-card", "feed-card", "card-item", "card"]):
|
||||
return "视频卡片"
|
||||
if "main" in features or "content" in features:
|
||||
return "主要内容区"
|
||||
if tag_name in ("input", "textarea"):
|
||||
return "输入控件"
|
||||
if tag_name == "button":
|
||||
return "按钮"
|
||||
if tag_name == "select":
|
||||
return "下拉选择"
|
||||
if tag_name in ("div", "li", "span", "img", "td", "label"):
|
||||
if any(w in features for w in ["btn", "button", "action", "clickable", "tab", "menu-item"]):
|
||||
return "按钮"
|
||||
if any(w in features for w in ["card", "item", "cell", "list-item"]):
|
||||
return "卡片/列表项"
|
||||
if "dropdown" in features or "menu" in features:
|
||||
return "菜单"
|
||||
if "dialog" in features or "modal" in features:
|
||||
return "弹窗"
|
||||
if "icon" in features:
|
||||
return "图标"
|
||||
if tag_name == "img":
|
||||
return "图片"
|
||||
return "交互区域"
|
||||
return "其他"
|
||||
|
||||
|
||||
# -------------------- 控件提取（去重 + 仅保留元素大小） --------------------
|
||||
def extract_controls(driver) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
无头模式下提取所有可交互元素（去重），
|
||||
仅返回元素的尺寸（CSS 像素），不包含坐标。
|
||||
"""
|
||||
raw = driver.execute_script("""
|
||||
const tags = ['a', 'input', 'button', 'select', 'textarea',
|
||||
'div', 'li', 'span', 'img', 'td', 'label'];
|
||||
let allElements = [];
|
||||
|
||||
// 1. 收集候选元素
|
||||
for (let tag of tags) {
|
||||
let elems = document.querySelectorAll(tag);
|
||||
for (let elem of elems) {
|
||||
if (tag === 'input' && elem.type === 'hidden') continue;
|
||||
if (tag === 'a' && !elem.href && !elem.onclick) continue;
|
||||
|
||||
let isInteractive = false;
|
||||
if (['div', 'li', 'span', 'img', 'td', 'label'].includes(tag)) {
|
||||
if (elem.onclick) isInteractive = true;
|
||||
let role = elem.getAttribute('role');
|
||||
if (role && ['button', 'link', 'tab', 'menuitem'].includes(role)) isInteractive = true;
|
||||
if (elem.hasAttribute('tabindex') && elem.getAttribute('tabindex') !== '-1') isInteractive = true;
|
||||
if (elem.hasAttribute('data-action') || elem.hasAttribute('data-href')) isInteractive = true;
|
||||
let cls = elem.className || '';
|
||||
if (/(btn|button|clickable|action|link|tab|menu-item|card|cell)/i.test(cls)) {
|
||||
isInteractive = true;
|
||||
}
|
||||
if (!isInteractive) continue;
|
||||
}
|
||||
allElements.push(elem);
|
||||
}
|
||||
],
|
||||
"max_tokens": 512,
|
||||
"temperature": 0
|
||||
}
|
||||
}
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
response = requests.post(API_URL, headers=headers, json=payload, timeout=120)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
// 2. 去重：只保留不被其他任何元素包含的最外层元素
|
||||
let uniqueElements = [];
|
||||
for (let elem of allElements) {
|
||||
let isChild = false;
|
||||
for (let other of allElements) {
|
||||
if (other !== elem && other.contains(elem)) {
|
||||
isChild = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!isChild) {
|
||||
uniqueElements.push(elem);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 获取每个元素的属性、文本和尺寸
|
||||
return uniqueElements.map(elem => {
|
||||
let tag = elem.tagName.toLowerCase();
|
||||
let attrs = {};
|
||||
for (let attr of elem.attributes) {
|
||||
attrs[attr.name] = attr.value;
|
||||
}
|
||||
let formAction = null;
|
||||
if (tag === 'button' || (tag === 'input' && ['submit','button','image'].includes(elem.type))) {
|
||||
let form = elem.closest('form');
|
||||
if (form) formAction = form.getAttribute('action') || '';
|
||||
}
|
||||
let text = elem.innerText ? elem.innerText.trim().substring(0, 50) : '';
|
||||
let rect = elem.getBoundingClientRect();
|
||||
|
||||
return {
|
||||
tag: tag,
|
||||
attributes: attrs,
|
||||
text: text,
|
||||
form_action: formAction,
|
||||
is_interactive: true,
|
||||
size: {
|
||||
width: rect.width,
|
||||
height: rect.height
|
||||
}
|
||||
};
|
||||
});
|
||||
""")
|
||||
|
||||
controls = []
|
||||
for item in raw:
|
||||
ctrl = {
|
||||
"type": item["tag"],
|
||||
"attributes": item["attributes"],
|
||||
"text": item.get("text", ""),
|
||||
"context": infer_context_from_attrs(item["tag"], item["attributes"], item.get("text", "")),
|
||||
"form_action": item.get("form_action"),
|
||||
"is_interactive": item.get("is_interactive", False),
|
||||
"size": item.get("size", {})
|
||||
}
|
||||
controls.append(ctrl)
|
||||
return controls
|
||||
|
||||
|
||||
# -------------------- 人性化报告生成 --------------------
|
||||
def short_text_for_control(ctrl: Dict[str, Any]) -> str:
|
||||
tp = ctrl["type"]
|
||||
attrs = ctrl["attributes"]
|
||||
is_interactive = ctrl.get("is_interactive", False)
|
||||
size = ctrl.get("size", {})
|
||||
size_str = ""
|
||||
if size:
|
||||
w, h = size.get("width", 0), size.get("height", 0)
|
||||
size_str = f" [大小:{w:.0f}x{h:.0f}]"
|
||||
|
||||
base = ""
|
||||
if tp == "a":
|
||||
href = attrs.get("href", "")
|
||||
text = ctrl.get("text") or attrs.get("title") or attrs.get("aria-label") or href[:60]
|
||||
base = f"链接 →{text}"
|
||||
if href:
|
||||
base += f" : {href}"
|
||||
elif tp == "button":
|
||||
text = ctrl.get("text") or attrs.get("value") or attrs.get("aria-label") or "按钮"
|
||||
base = f"按钮: {text}"
|
||||
elif tp == "input":
|
||||
input_type = attrs.get("type", "text")
|
||||
text = ctrl.get("text") or attrs.get("value") or attrs.get("placeholder") or ""
|
||||
base = f"输入框 ({input_type})"
|
||||
if text:
|
||||
base += f" [{text}]"
|
||||
elif tp == "textarea":
|
||||
text = ctrl.get("text") or attrs.get("placeholder") or ""
|
||||
base = "多行文本框"
|
||||
if text:
|
||||
base += f" [{text}]"
|
||||
elif tp == "select":
|
||||
text = ctrl.get("text") or ""
|
||||
base = "下拉选择框"
|
||||
if text:
|
||||
base += f" [{text}]"
|
||||
elif is_interactive:
|
||||
text = ctrl.get("text") or ""
|
||||
role = attrs.get("role", "")
|
||||
tag_desc = f"{tp}控件"
|
||||
if role == "button":
|
||||
tag_desc = "按钮(ARIA)"
|
||||
elif role == "link":
|
||||
tag_desc = "链接(ARIA)"
|
||||
elif "card" in (attrs.get("class", "") + attrs.get("id", "")):
|
||||
tag_desc = "卡片"
|
||||
base = f"可交互{tag_desc}"
|
||||
if text:
|
||||
base += f" [{text}]"
|
||||
if attrs.get("data-href"):
|
||||
base += f" (目标: {attrs['data-href']})"
|
||||
else:
|
||||
text = ctrl.get("text") or ""
|
||||
base = f"{tp} 控件"
|
||||
if text:
|
||||
base += f" [{text}]"
|
||||
return base + size_str
|
||||
|
||||
|
||||
def generate_report(url: str, controls: List[Dict[str, Any]]) -> str:
|
||||
lines = [f"🌐 页面交互控件分析报告（静态单页·无头模式）\n页面: {url}\n" + "=" * 50]
|
||||
groups: Dict[str, List[Dict]] = {}
|
||||
for ctrl in controls:
|
||||
ctx = ctrl["context"]
|
||||
groups.setdefault(ctx, []).append(ctrl)
|
||||
|
||||
sorted_groups = sorted(groups.items(), key=lambda x: x[0])
|
||||
for context, items in sorted_groups:
|
||||
lines.append(f"├── 【{context}】({len(items)}个)")
|
||||
for item in items:
|
||||
desc = short_text_for_control(item)
|
||||
lines.append(f"│ ├── {desc}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# -------------------- 主程序 --------------------
|
||||
def main():
|
||||
# 1. 缩放图片
|
||||
resize_image_force(IMAGE_PATH, RESIZED_PATH, TARGET_SIZE)
|
||||
parser = argparse.ArgumentParser(description="静态单页交互控件分析（Edge·无头模式·仅元素大小）")
|
||||
parser.add_argument("source", help="起始 URL 或本地 HTML 文件路径")
|
||||
parser.add_argument("-o", "--output", help="输出报告文件路径，默认打印到控制台")
|
||||
parser.add_argument("--json", action="store_true", help="同时保存机器可读的 JSON 文件（与 -o 同名 .json）")
|
||||
args = parser.parse_args()
|
||||
|
||||
# 2. 调用模型
|
||||
print("正在调用模型，请稍候...")
|
||||
result = call_model(RESIZED_PATH, INSTRUCTION)
|
||||
# 构造 URL
|
||||
if args.source.startswith(('http://', 'https://')):
|
||||
url = args.source
|
||||
else:
|
||||
abs_path = os.path.abspath(args.source)
|
||||
url = Path(abs_path).as_uri()
|
||||
|
||||
# 无头模式 Edge，设置窗口大小
|
||||
options = Options()
|
||||
options.add_argument("--headless=new") # 新版无头模式
|
||||
options.add_argument("--window-size=1920,1080") # 固定视口大小
|
||||
options.add_argument("--disable-gpu") # 禁用 GPU 加速（无头下建议）
|
||||
|
||||
service = Service(EdgeChromiumDriverManager().install())
|
||||
driver = webdriver.Edge(service=service, options=options)
|
||||
|
||||
# 3. 提取回复文本
|
||||
try:
|
||||
content = result["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError) as e:
|
||||
print(f"解析响应出错：{e}")
|
||||
print("原始响应：", json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return
|
||||
driver.get(url)
|
||||
WebDriverWait(driver, 15).until(
|
||||
lambda d: d.execute_script("return document.readyState") == "complete"
|
||||
)
|
||||
controls = extract_controls(driver)
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
# 4. 保存原始回复为 UTF-8 文件
|
||||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
print(f"模型回复已保存至: {OUTPUT_FILE}")
|
||||
report = generate_report(url, controls)
|
||||
|
||||
# 输出结果
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(report)
|
||||
print(f"人性化报告已保存到 {args.output}")
|
||||
if args.json:
|
||||
json_path = args.output.rsplit('.', 1)[0] + '.json'
|
||||
with open(json_path, 'w', encoding='utf-8') as jf:
|
||||
json.dump({
|
||||
"url": url,
|
||||
"controls": controls
|
||||
}, jf, ensure_ascii=False, indent=2)
|
||||
print(f"机器可读 JSON 已保存到 {json_path}")
|
||||
else:
|
||||
print(report)
|
||||
|
||||
# 5. 直接打印（控制台可能显示乱码，但文件正确即可）
|
||||
print("\n--- 模型回复（原始内容）---")
|
||||
print(content)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
Reference in new issue
Block a user