lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-03-15 22:48:40 +08:00
parent 2de8a08f2c e166106b6a
commit 6face585f3
2 changed files with 17 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
desktop_env/server/main.py
View File

@@ -284,6 +284,15 @@ def _create_atspi_node(node: Accessible, depth: int = 0, flag: Optional[str] = N
text = text.replace("\ufffc", "").replace("\ufffd", "") text = text.replace("\ufffc", "").replace("\ufffd", "")
# }}} Text # # }}} Text #
# Image {{{ #
try:
node.queryImage()
except NotImplementedError:
pass
else:
attribute_dict["image"] = "true"
# }}} Image #
# Selection {{{ # # Selection {{{ #
try: try:
node.querySelection() node.querySelection()

14
mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
View File

@@ -26,7 +26,7 @@ def find_leaf_nodes(xlm_file_str):
state_ns = "uri:deskat:state.at-spi.gnome.org" state_ns = "uri:deskat:state.at-spi.gnome.org"
component_ns = "uri:deskat:component.at-spi.gnome.org" component_ns = "uri:deskat:component.at-spi.gnome.org"
def judge_node(node: ET, platform="ubuntu") -> bool: def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
keeps: bool = node.tag.startswith("document")\ keeps: bool = node.tag.startswith("document")\
or node.tag.endswith("item")\ or node.tag.endswith("item")\
or node.tag.endswith("button")\ or node.tag.endswith("button")\
@@ -60,7 +60,9 @@ def judge_node(node: ET, platform="ubuntu") -> bool:
or node.get("{{{:}}}expandable".format(state_ns), "false")=="true"\ or node.get("{{{:}}}expandable".format(state_ns), "false")=="true"\
or node.get("{{{:}}}checkable".format(state_ns), "false")=="true" or node.get("{{{:}}}checkable".format(state_ns), "false")=="true"
)\ )\
and (node.get("name", "") != "" or node.text is not None and len(node.text)>0) and ( node.get("name", "") != "" or node.text is not None and len(node.text)>0\
or check_image and node.get("image", "false")=="true"
)
coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format(component_ns), "(-1, -1)")) coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format(component_ns), "(-1, -1)"))
sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format(component_ns), "(-1, -1)")) sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format(component_ns), "(-1, -1)"))
@@ -155,12 +157,12 @@ def print_nodes_with_indent(nodes, indent=0):
if __name__ == '__main__': if __name__ == '__main__':
import json import json
with open('4.json', 'r', encoding='utf-8') as f: with open('selection_sorted(imaged).xml', 'r', encoding='utf-8') as f:
xml_file_str = json.load(f)["AT"] xml_file_str = f.read()
filtered_nodes = filter_nodes(ET.fromstring(xml_file_str)) filtered_nodes = filter_nodes(ET.fromstring(xml_file_str))
print(len(filtered_nodes)) print(len(filtered_nodes))
masks = draw_bounding_boxes( filtered_nodes, '4.png' masks = draw_bounding_boxes( filtered_nodes, 'selection_sorted(imaged).png'
, '4.a.png' , 'selection_sorted(imaged).ai.png'
) )
# print(masks) # print(masks)