feat: 增强科研软件的 a11y tree 支持
- 扩展 heuristic_retrieve.py 白名单以覆盖科研软件 GUI 框架:
- 新增 prefix 规则: sunawt (Java Swing), qt5q/qt6q (Qt), ovito, pymol,
contentspanel, wx (wxWidgets), afx (MFC), thunderrt (VB6)
- 新增 endswith 规则: edit, widget, box, dialog, view, frame, menuitem,
menubar, toolbar, tabitem, treeitem, window
- 新增 Qt 控件和 Win32 控件的精确匹配
- 在 agent.py 中添加原始 a11y tree 的调试日志
- 修复 run.py 中 agent 初始化缺少 platform='windows' 的问题
- 添加 NO_PROXY 绕过本地/VM IP (兼容 Clash 全局代理)
- lib_run_single.py 中应用启动等待时间增加到 15 秒
- 新增 test_each_domain_a11y_tree.json (每个域一个任务用于 a11y 验证)
This commit is contained in:
9
evaluation_examples/test_each_domain_a11y_tree.json
Normal file
9
evaluation_examples/test_each_domain_a11y_tree.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"avogadro": ["building-organic-molecules_task1"],
|
||||
"imagej": ["user-guide_task1"],
|
||||
"jade": ["MDIJade6.5使用手册_task1"],
|
||||
"origin": ["Origin_User_Guide_2025b_E_task1"],
|
||||
"ovito": ["animation_task3"],
|
||||
"pymol": ["MovieSchool_1_task1"],
|
||||
"vesta": ["VESTA_Manual_task1"]
|
||||
}
|
||||
@@ -22,7 +22,7 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
|
||||
# except Exception as e:
|
||||
# agent.reset(vm_ip=env.vm_ip)
|
||||
|
||||
# time.sleep(10) # Wait for the environment to be ready
|
||||
time.sleep(15) # Wait for the environment to be ready (apps like Avogadro need time to fully load)
|
||||
|
||||
# get initial observation
|
||||
logger.info("Getting initial observation...")
|
||||
|
||||
@@ -46,6 +46,15 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
|
||||
raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
|
||||
|
||||
keeps: bool = node.tag.startswith("document") \
|
||||
or node.tag.startswith("sunawt") \
|
||||
or node.tag.startswith("qt5q") \
|
||||
or node.tag.startswith("qt6q") \
|
||||
or node.tag.startswith("ovito") \
|
||||
or node.tag.startswith("pymol") \
|
||||
or node.tag.startswith("contentspanel") \
|
||||
or node.tag.startswith("wx") \
|
||||
or node.tag.startswith("afx") \
|
||||
or node.tag.startswith("thunderrt") \
|
||||
or node.tag.endswith("item") \
|
||||
or node.tag.endswith("button") \
|
||||
or node.tag.endswith("heading") \
|
||||
@@ -58,6 +67,18 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
|
||||
or node.tag.endswith("textfield") \
|
||||
or node.tag.endswith("textarea") \
|
||||
or node.tag.endswith("menu") \
|
||||
or node.tag.endswith("menuitem") \
|
||||
or node.tag.endswith("menubar") \
|
||||
or node.tag.endswith("toolbar") \
|
||||
or node.tag.endswith("tabitem") \
|
||||
or node.tag.endswith("treeitem") \
|
||||
or node.tag.endswith("window") \
|
||||
or node.tag.endswith("edit") \
|
||||
or node.tag.endswith("widget") \
|
||||
or node.tag.endswith("box") \
|
||||
or node.tag.endswith("dialog") \
|
||||
or node.tag.endswith("view") \
|
||||
or node.tag.endswith("frame") \
|
||||
or node.tag in {"alert", "canvas", "check-box"
|
||||
, "combo-box", "entry", "icon"
|
||||
, "image", "paragraph", "scroll-bar"
|
||||
@@ -66,6 +87,16 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
|
||||
, "netuiribbontab", "start", "trayclockwclass"
|
||||
, "traydummysearchcontrol", "uiimage", "uiproperty"
|
||||
, "uiribboncommandbar"
|
||||
, "qt5qwindowicon", "textblock", "listview"
|
||||
, "chrome_widgetwin_1", "chrome_renderwidgethosthwnd"
|
||||
, "unknown", "pane", "tree", "tab"
|
||||
, "datagrid", "dataitem", "group"
|
||||
, "statusbar", "titlebar", "tooltip"
|
||||
, "toolbarwindow32", "richedit50w"
|
||||
, "msctls_statusbar32", "qaction"
|
||||
, "qsplitter", "qsplitterhandle"
|
||||
, "qtoolbarseparator", "qtextbrowser"
|
||||
, "qtabbar", "qopenglwidget"
|
||||
}
|
||||
keeps = keeps and (
|
||||
platform == "ubuntu"
|
||||
|
||||
@@ -502,9 +502,30 @@ class PromptAgent:
|
||||
]
|
||||
})
|
||||
elif self.observation_type == "a11y_tree":
|
||||
# Debug: log raw a11y tree XML to help diagnose missing elements
|
||||
raw_tree = obs["accessibility_tree"]
|
||||
if raw_tree:
|
||||
# Log first 2000 chars of raw XML and count total nodes
|
||||
root = ET.fromstring(raw_tree)
|
||||
all_tags = set()
|
||||
total_nodes = 0
|
||||
for node in root.iter():
|
||||
all_tags.add(node.tag)
|
||||
total_nodes += 1
|
||||
logger.info("Raw a11y tree: %d total nodes, unique tags: %s", total_nodes, all_tags)
|
||||
logger.debug("Raw a11y tree XML (first 2000 chars): %s", raw_tree[:2000])
|
||||
# Also log nodes containing 'avogadro' or 'qt5' in their attributes
|
||||
for node in root.iter():
|
||||
node_str = ET.tostring(node, encoding="unicode")
|
||||
if 'avogadro' in node_str.lower() or 'qt5' in node_str.lower():
|
||||
logger.info("Avogadro/Qt5 node: tag=%s, name=%s, visible=%s, enabled=%s",
|
||||
node.tag, node.get("name", ""),
|
||||
node.get("{https://accessibility.windows.example.org/ns/state}visible", "?"),
|
||||
node.get("{https://accessibility.windows.example.org/ns/state}enabled", "?"))
|
||||
linearized_accessibility_tree = linearize_accessibility_tree(accessibility_tree=obs["accessibility_tree"],
|
||||
platform=self.platform)
|
||||
logger.debug("LINEAR AT: %s", linearized_accessibility_tree)
|
||||
logger.info("Linearized a11y tree lines: %d", len(linearized_accessibility_tree.split('\n')) if linearized_accessibility_tree else 0)
|
||||
|
||||
if linearized_accessibility_tree:
|
||||
linearized_accessibility_tree = trim_accessibility_tree(linearized_accessibility_tree,
|
||||
@@ -670,7 +691,7 @@ class PromptAgent:
|
||||
return response.json()['choices'][0]['message']['content']
|
||||
elif self.model.startswith("gpt"):
|
||||
# Support custom OpenAI base URL via environment variable
|
||||
base_url = os.environ.get('OPENAI_BASE_URL', 'https://api.openai.com')
|
||||
base_url = os.environ.get('OPENAI_BASE_URL', os.environ.get('OPENAI_API_BASE', 'https://api.openai.com'))
|
||||
# Smart handling: avoid duplicate /v1 if base_url already ends with /v1
|
||||
api_url = f"{base_url}/chat/completions" if base_url.endswith('/v1') else f"{base_url}/v1/chat/completions"
|
||||
headers = {
|
||||
|
||||
12
run.py
12
run.py
@@ -142,6 +142,7 @@ def test(args: argparse.Namespace, test_all_meta: dict) -> None:
|
||||
}
|
||||
|
||||
agent = PromptAgent(
|
||||
platform="windows",
|
||||
model=args.model,
|
||||
max_tokens=args.max_tokens,
|
||||
top_p=args.top_p,
|
||||
@@ -321,6 +322,17 @@ def get_result(action_space, use_model, observation_type, result_dir, total_file
|
||||
if __name__ == "__main__":
|
||||
####### The complete version of the list of examples #######
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
|
||||
# Bypass proxy for local/VM IPs so Clash global mode doesn't break VM communication
|
||||
no_proxy = os.environ.get("NO_PROXY", os.environ.get("no_proxy", ""))
|
||||
extra = "localhost,127.0.0.1,192.168.*,10.*,172.16.*,172.17.*,172.18.*,172.19.*,172.20.*,172.21.*,172.22.*,172.23.*,172.24.*,172.25.*,172.26.*,172.27.*,172.28.*,172.29.*,172.30.*,172.31.*"
|
||||
if no_proxy:
|
||||
no_proxy = f"{no_proxy},{extra}"
|
||||
else:
|
||||
no_proxy = extra
|
||||
os.environ["NO_PROXY"] = no_proxy
|
||||
os.environ["no_proxy"] = no_proxy
|
||||
|
||||
args = config()
|
||||
|
||||
# save args to json in result_dir/action_space/observation_type/model/args.json
|
||||
|
||||
Reference in New Issue
Block a user