389 lines
15 KiB
Python
389 lines
15 KiB
Python
# import os
|
||
# import re
|
||
# import json
|
||
# import argparse
|
||
# from pathlib import Path
|
||
# from typing import Dict, Any, List, Optional
|
||
# from urllib.parse import urljoin, urlparse
|
||
|
||
# from selenium import webdriver
|
||
# from selenium.webdriver.edge.options import Options
|
||
# from selenium.webdriver.edge.service import Service
|
||
# from selenium.webdriver.support.ui import WebDriverWait
|
||
# from webdriver_manager.microsoft import EdgeChromiumDriverManager
|
||
|
||
|
||
# # -------------------- 工具函数 --------------------
|
||
# def get_absolute_url(base_url: str, href: str) -> Optional[str]:
|
||
# if not href:
|
||
# return None
|
||
# href = href.strip()
|
||
# if href.startswith(('javascript:', 'mailto:', 'tel:', '#')):
|
||
# return None
|
||
# absolute = urljoin(base_url, href)
|
||
# parsed = urlparse(absolute)
|
||
# if parsed.scheme in ('http', 'https', 'file'):
|
||
# return absolute
|
||
# return None
|
||
|
||
|
||
# # -------------------- 语义上下文推断 --------------------
|
||
# def infer_context_from_attrs(tag_name: str, attrs: Dict[str, str], text: str = "") -> str:
|
||
# class_list = attrs.get("class", "").split()
|
||
# id_ = attrs.get("id", "")
|
||
# href = attrs.get("href", "")
|
||
# features = " ".join(class_list) + " " + id_
|
||
|
||
# if tag_name == "a":
|
||
# if any(w in features for w in ["btn", "button", "tab", "nav-btn", "menu-item", "action-btn"]):
|
||
# return "按钮"
|
||
# if "cm.bilibili.com" in href or "ad" in features:
|
||
# return "广告链接"
|
||
# return "普通链接"
|
||
|
||
# if "search" in features or "nav-search" in features:
|
||
# return "搜索框"
|
||
# if any(w in features for w in ["header", "top-bar", "masthead", "nav"]):
|
||
# return "顶部导航"
|
||
# if any(w in features for w in ["sidebar", "aside", "right-entry"]):
|
||
# return "侧边栏"
|
||
# if "footer" in features:
|
||
# return "页脚"
|
||
# if any(w in features for w in ["channel", "partition", "channel-icons"]):
|
||
# return "频道/分区"
|
||
# if any(w in features for w in ["carousel", "banner", "slider"]):
|
||
# return "轮播/横幅"
|
||
# if any(w in features for w in ["video-card", "feed-card", "card-item", "card"]):
|
||
# return "视频卡片"
|
||
# if "main" in features or "content" in features:
|
||
# return "主要内容区"
|
||
# if tag_name in ("input", "textarea"):
|
||
# return "输入控件"
|
||
# if tag_name == "button":
|
||
# return "按钮"
|
||
# if tag_name == "select":
|
||
# return "下拉选择"
|
||
# if tag_name in ("div", "li", "span", "img", "td", "label"):
|
||
# if any(w in features for w in ["btn", "button", "action", "clickable", "tab", "menu-item"]):
|
||
# return "按钮"
|
||
# if any(w in features for w in ["card", "item", "cell", "list-item"]):
|
||
# return "卡片/列表项"
|
||
# if "dropdown" in features or "menu" in features:
|
||
# return "菜单"
|
||
# if "dialog" in features or "modal" in features:
|
||
# return "弹窗"
|
||
# if "icon" in features:
|
||
# return "图标"
|
||
# if tag_name == "img":
|
||
# return "图片"
|
||
# return "交互区域"
|
||
# return "其他"
|
||
|
||
|
||
# # -------------------- 控件提取（去重 + 仅保留元素大小） --------------------
|
||
# def extract_controls(driver) -> List[Dict[str, Any]]:
|
||
# """
|
||
# 无头模式下提取所有可交互元素（去重），
|
||
# 仅返回元素的尺寸（CSS 像素），不包含坐标。
|
||
# """
|
||
# raw = driver.execute_script("""
|
||
# const tags = ['a', 'input', 'button', 'select', 'textarea',
|
||
# 'div', 'li', 'span', 'img', 'td', 'label'];
|
||
# let allElements = [];
|
||
|
||
# // 1. 收集候选元素
|
||
# for (let tag of tags) {
|
||
# let elems = document.querySelectorAll(tag);
|
||
# for (let elem of elems) {
|
||
# if (tag === 'input' && elem.type === 'hidden') continue;
|
||
# if (tag === 'a' && !elem.href && !elem.onclick) continue;
|
||
|
||
# let isInteractive = false;
|
||
# if (['div', 'li', 'span', 'img', 'td', 'label'].includes(tag)) {
|
||
# if (elem.onclick) isInteractive = true;
|
||
# let role = elem.getAttribute('role');
|
||
# if (role && ['button', 'link', 'tab', 'menuitem'].includes(role)) isInteractive = true;
|
||
# if (elem.hasAttribute('tabindex') && elem.getAttribute('tabindex') !== '-1') isInteractive = true;
|
||
# if (elem.hasAttribute('data-action') || elem.hasAttribute('data-href')) isInteractive = true;
|
||
# let cls = elem.className || '';
|
||
# if (/(btn|button|clickable|action|link|tab|menu-item|card|cell)/i.test(cls)) {
|
||
# isInteractive = true;
|
||
# }
|
||
# if (!isInteractive) continue;
|
||
# }
|
||
# allElements.push(elem);
|
||
# }
|
||
# }
|
||
|
||
# // 2. 去重：只保留不被其他任何元素包含的最外层元素
|
||
# let uniqueElements = [];
|
||
# for (let elem of allElements) {
|
||
# let isChild = false;
|
||
# for (let other of allElements) {
|
||
# if (other !== elem && other.contains(elem)) {
|
||
# isChild = true;
|
||
# break;
|
||
# }
|
||
# }
|
||
# if (!isChild) {
|
||
# uniqueElements.push(elem);
|
||
# }
|
||
# }
|
||
|
||
# // 3. 获取每个元素的属性、文本和尺寸
|
||
# return uniqueElements.map(elem => {
|
||
# let tag = elem.tagName.toLowerCase();
|
||
# let attrs = {};
|
||
# for (let attr of elem.attributes) {
|
||
# attrs[attr.name] = attr.value;
|
||
# }
|
||
# let formAction = null;
|
||
# if (tag === 'button' || (tag === 'input' && ['submit','button','image'].includes(elem.type))) {
|
||
# let form = elem.closest('form');
|
||
# if (form) formAction = form.getAttribute('action') || '';
|
||
# }
|
||
# let text = elem.innerText ? elem.innerText.trim().substring(0, 50) : '';
|
||
# let rect = elem.getBoundingClientRect();
|
||
|
||
# return {
|
||
# tag: tag,
|
||
# attributes: attrs,
|
||
# text: text,
|
||
# form_action: formAction,
|
||
# is_interactive: true,
|
||
# size: {
|
||
# width: rect.width,
|
||
# height: rect.height
|
||
# }
|
||
# };
|
||
# });
|
||
# """)
|
||
|
||
# controls = []
|
||
# for item in raw:
|
||
# ctrl = {
|
||
# "type": item["tag"],
|
||
# "attributes": item["attributes"],
|
||
# "text": item.get("text", ""),
|
||
# "context": infer_context_from_attrs(item["tag"], item["attributes"], item.get("text", "")),
|
||
# "form_action": item.get("form_action"),
|
||
# "is_interactive": item.get("is_interactive", False),
|
||
# "size": item.get("size", {})
|
||
# }
|
||
# controls.append(ctrl)
|
||
# return controls
|
||
|
||
|
||
# # -------------------- 人性化报告生成 --------------------
|
||
# def short_text_for_control(ctrl: Dict[str, Any]) -> str:
|
||
# tp = ctrl["type"]
|
||
# attrs = ctrl["attributes"]
|
||
# is_interactive = ctrl.get("is_interactive", False)
|
||
# size = ctrl.get("size", {})
|
||
# size_str = ""
|
||
# if size:
|
||
# w, h = size.get("width", 0), size.get("height", 0)
|
||
# size_str = f" [大小:{w:.0f}x{h:.0f}]"
|
||
|
||
# base = ""
|
||
# if tp == "a":
|
||
# href = attrs.get("href", "")
|
||
# text = ctrl.get("text") or attrs.get("title") or attrs.get("aria-label") or href[:60]
|
||
# base = f"链接 →{text}"
|
||
# if href:
|
||
# base += f" : {href}"
|
||
# elif tp == "button":
|
||
# text = ctrl.get("text") or attrs.get("value") or attrs.get("aria-label") or "按钮"
|
||
# base = f"按钮: {text}"
|
||
# elif tp == "input":
|
||
# input_type = attrs.get("type", "text")
|
||
# text = ctrl.get("text") or attrs.get("value") or attrs.get("placeholder") or ""
|
||
# base = f"输入框 ({input_type})"
|
||
# if text:
|
||
# base += f" [{text}]"
|
||
# elif tp == "textarea":
|
||
# text = ctrl.get("text") or attrs.get("placeholder") or ""
|
||
# base = "多行文本框"
|
||
# if text:
|
||
# base += f" [{text}]"
|
||
# elif tp == "select":
|
||
# text = ctrl.get("text") or ""
|
||
# base = "下拉选择框"
|
||
# if text:
|
||
# base += f" [{text}]"
|
||
# elif is_interactive:
|
||
# text = ctrl.get("text") or ""
|
||
# role = attrs.get("role", "")
|
||
# tag_desc = f"{tp}控件"
|
||
# if role == "button":
|
||
# tag_desc = "按钮(ARIA)"
|
||
# elif role == "link":
|
||
# tag_desc = "链接(ARIA)"
|
||
# elif "card" in (attrs.get("class", "") + attrs.get("id", "")):
|
||
# tag_desc = "卡片"
|
||
# base = f"可交互{tag_desc}"
|
||
# if text:
|
||
# base += f" [{text}]"
|
||
# if attrs.get("data-href"):
|
||
# base += f" (目标: {attrs['data-href']})"
|
||
# else:
|
||
# text = ctrl.get("text") or ""
|
||
# base = f"{tp} 控件"
|
||
# if text:
|
||
# base += f" [{text}]"
|
||
# return base + size_str
|
||
|
||
|
||
# def generate_report(url: str, controls: List[Dict[str, Any]]) -> str:
|
||
# lines = [f"🌐 页面交互控件分析报告（静态单页·无头模式）\n页面: {url}\n" + "=" * 50]
|
||
# groups: Dict[str, List[Dict]] = {}
|
||
# for ctrl in controls:
|
||
# ctx = ctrl["context"]
|
||
# groups.setdefault(ctx, []).append(ctrl)
|
||
|
||
# sorted_groups = sorted(groups.items(), key=lambda x: x[0])
|
||
# for context, items in sorted_groups:
|
||
# lines.append(f"├── 【{context}】({len(items)}个)")
|
||
# for item in items:
|
||
# desc = short_text_for_control(item)
|
||
# lines.append(f"│ ├── {desc}")
|
||
# return "\n".join(lines)
|
||
|
||
|
||
# # -------------------- 主程序 --------------------
|
||
# def main():
|
||
# parser = argparse.ArgumentParser(description="静态单页交互控件分析（Edge·无头模式·仅元素大小）")
|
||
# parser.add_argument("source", help="起始 URL 或本地 HTML 文件路径")
|
||
# parser.add_argument("-o", "--output", help="输出报告文件路径，默认打印到控制台")
|
||
# parser.add_argument("--json", action="store_true", help="同时保存机器可读的 JSON 文件（与 -o 同名 .json）")
|
||
# args = parser.parse_args()
|
||
|
||
# # 构造 URL
|
||
# if args.source.startswith(('http://', 'https://')):
|
||
# url = args.source
|
||
# else:
|
||
# abs_path = os.path.abspath(args.source)
|
||
# url = Path(abs_path).as_uri()
|
||
|
||
# # 无头模式 Edge，设置窗口大小
|
||
# options = Options()
|
||
# options.add_argument("--headless=new") # 新版无头模式
|
||
# options.add_argument("--window-size=1920,1080") # 固定视口大小
|
||
# options.add_argument("--disable-gpu") # 禁用 GPU 加速（无头下建议）
|
||
|
||
# service = Service(EdgeChromiumDriverManager().install())
|
||
# driver = webdriver.Edge(service=service, options=options)
|
||
|
||
# try:
|
||
# driver.get(url)
|
||
# WebDriverWait(driver, 15).until(
|
||
# lambda d: d.execute_script("return document.readyState") == "complete"
|
||
# )
|
||
# controls = extract_controls(driver)
|
||
# finally:
|
||
# driver.quit()
|
||
|
||
# report = generate_report(url, controls)
|
||
|
||
# # 输出结果
|
||
# if args.output:
|
||
# with open(args.output, 'w', encoding='utf-8') as f:
|
||
# f.write(report)
|
||
# print(f"人性化报告已保存到 {args.output}")
|
||
# if args.json:
|
||
# json_path = args.output.rsplit('.', 1)[0] + '.json'
|
||
# with open(json_path, 'w', encoding='utf-8') as jf:
|
||
# json.dump({
|
||
# "url": url,
|
||
# "controls": controls
|
||
# }, jf, ensure_ascii=False, indent=2)
|
||
# print(f"机器可读 JSON 已保存到 {json_path}")
|
||
# else:
|
||
# print(report)
|
||
|
||
|
||
# if __name__ == "__main__":
|
||
# main()
|
||
import base64
|
||
import io
|
||
import json
|
||
import requests
|
||
from PIL import Image
|
||
|
||
# ================= 配置 =================
|
||
IMAGE_PATH = r"C:\Users\86166\Desktop\3.png"
|
||
RESIZED_PATH = r"C:\Users\86166\Desktop\3_resized.png"
|
||
OUTPUT_FILE = r"C:\Users\86166\Desktop\model_output.txt"
|
||
API_URL = "http://localhost:4555/v1/chat/completions"
|
||
MODEL_NAME = "UI-TARS-1.5-7B"
|
||
TARGET_SIZE = (1000, 1000) # 宽, 高
|
||
INSTRUCTION = "点击屏幕中央。请只输出点击的x和y坐标，格式为'x,y'，不要输出任何其他内容。"
|
||
# =======================================
|
||
|
||
def resize_image_force(image_path, save_path, size):
|
||
"""强制拉伸图片到指定尺寸（不保持比例）"""
|
||
img = Image.open(image_path).convert("RGB")
|
||
img_resized = img.resize(size, Image.Resampling.LANCZOS)
|
||
img_resized.save(save_path, "PNG")
|
||
print(f"图片已缩放至 {size[0]}×{size[1]}，保存至: {save_path}")
|
||
return save_path
|
||
|
||
def image_to_base64(image_path):
|
||
"""将图片转为 base64 字符串（去除换行）"""
|
||
with open(image_path, "rb") as f:
|
||
img_bytes = f.read()
|
||
b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||
return b64
|
||
|
||
def call_model(image_path, instruction):
|
||
"""调用 vLLM 服务"""
|
||
b64 = image_to_base64(image_path)
|
||
data_url = f"data:image/png;base64,{b64}"
|
||
|
||
payload = {
|
||
"model": MODEL_NAME,
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "image_url", "image_url": {"url": data_url}},
|
||
{"type": "text", "text": instruction}
|
||
]
|
||
}
|
||
],
|
||
"max_tokens": 512,
|
||
"temperature": 0
|
||
}
|
||
|
||
headers = {"Content-Type": "application/json"}
|
||
response = requests.post(API_URL, headers=headers, json=payload, timeout=120)
|
||
response.raise_for_status()
|
||
return response.json()
|
||
|
||
def main():
|
||
# 1. 缩放图片
|
||
resize_image_force(IMAGE_PATH, RESIZED_PATH, TARGET_SIZE)
|
||
|
||
# 2. 调用模型
|
||
print("正在调用模型，请稍候...")
|
||
result = call_model(RESIZED_PATH, INSTRUCTION)
|
||
|
||
# 3. 提取回复文本
|
||
try:
|
||
content = result["choices"][0]["message"]["content"]
|
||
except (KeyError, IndexError) as e:
|
||
print(f"解析响应出错：{e}")
|
||
print("原始响应：", json.dumps(result, ensure_ascii=False, indent=2))
|
||
return
|
||
|
||
# 4. 保存原始回复为 UTF-8 文件
|
||
with open(OUTPUT_FILE, "w", encoding="utf-8") as f:
|
||
f.write(content)
|
||
print(f"模型回复已保存至: {OUTPUT_FILE}")
|
||
|
||
# 5. 直接打印（控制台可能显示乱码，但文件正确即可）
|
||
print("\n--- 模型回复（原始内容）---")
|
||
print(content)
|
||
|
||
if __name__ == "__main__":
|
||
main() |