feat: 增强科研软件的 a11y tree 支持

- 扩展 heuristic_retrieve.py 白名单以覆盖科研软件 GUI 框架:
  - 新增 prefix 规则: sunawt (Java Swing), qt5q/qt6q (Qt), ovito, pymol,
    contentspanel, wx (wxWidgets), afx (MFC), thunderrt (VB6)
  - 新增 endswith 规则: edit, widget, box, dialog, view, frame, menuitem,
    menubar, toolbar, tabitem, treeitem, window
  - 新增 Qt 控件和 Win32 控件的精确匹配
- 在 agent.py 中添加原始 a11y tree 的调试日志
- 修复 run.py 中 agent 初始化缺少 platform='windows' 的问题
- 添加 NO_PROXY 绕过本地/VM IP (兼容 Clash 全局代理)
- lib_run_single.py 中应用启动等待时间增加到 15 秒
- 新增 test_each_domain_a11y_tree.json (每个域一个任务用于 a11y 验证)
This commit is contained in:
2026-02-26 15:04:28 +08:00
parent 9899d4a0c7
commit 07e66490dd
5 changed files with 118 additions and 45 deletions

View File

@@ -46,6 +46,15 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
keeps: bool = node.tag.startswith("document") \
or node.tag.startswith("sunawt") \
or node.tag.startswith("qt5q") \
or node.tag.startswith("qt6q") \
or node.tag.startswith("ovito") \
or node.tag.startswith("pymol") \
or node.tag.startswith("contentspanel") \
or node.tag.startswith("wx") \
or node.tag.startswith("afx") \
or node.tag.startswith("thunderrt") \
or node.tag.endswith("item") \
or node.tag.endswith("button") \
or node.tag.endswith("heading") \
@@ -58,6 +67,18 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
or node.tag.endswith("textfield") \
or node.tag.endswith("textarea") \
or node.tag.endswith("menu") \
or node.tag.endswith("menuitem") \
or node.tag.endswith("menubar") \
or node.tag.endswith("toolbar") \
or node.tag.endswith("tabitem") \
or node.tag.endswith("treeitem") \
or node.tag.endswith("window") \
or node.tag.endswith("edit") \
or node.tag.endswith("widget") \
or node.tag.endswith("box") \
or node.tag.endswith("dialog") \
or node.tag.endswith("view") \
or node.tag.endswith("frame") \
or node.tag in {"alert", "canvas", "check-box"
, "combo-box", "entry", "icon"
, "image", "paragraph", "scroll-bar"
@@ -66,6 +87,16 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
, "netuiribbontab", "start", "trayclockwclass"
, "traydummysearchcontrol", "uiimage", "uiproperty"
, "uiribboncommandbar"
, "qt5qwindowicon", "textblock", "listview"
, "chrome_widgetwin_1", "chrome_renderwidgethosthwnd"
, "unknown", "pane", "tree", "tab"
, "datagrid", "dataitem", "group"
, "statusbar", "titlebar", "tooltip"
, "toolbarwindow32", "richedit50w"
, "msctls_statusbar32", "qaction"
, "qsplitter", "qsplitterhandle"
, "qtoolbarseparator", "qtextbrowser"
, "qtabbar", "qopenglwidget"
}
keeps = keeps and (
platform == "ubuntu"