lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Modify the namespace of a11y tree (#62)

Browse Source
This commit is contained in:
Tianbao Xie
2024-07-25 20:20:34 +08:00
committed by GitHub
parent 1fd8b66fde
commit a156f8a3d6
3 changed files with 118 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
View File

@@ -26,11 +26,25 @@ def find_leaf_nodes(xlm_file_str):
return leaf_nodes
state_ns = "uri:deskat:state.at-spi.gnome.org"
component_ns = "uri:deskat:component.at-spi.gnome.org"
state_ns_ubuntu = "https://accessibility.ubuntu.example.org/ns/state"
state_ns_windows = "https://accessibility.windows.example.org/ns/state"
component_ns_ubuntu = "https://accessibility.ubuntu.example.org/ns/component"
component_ns_windows = "https://accessibility.windows.example.org/ns/component"
value_ns_ubuntu = "https://accessibility.ubuntu.example.org/ns/value"
value_ns_windows = "https://accessibility.windows.example.org/ns/value"
class_ns_windows = "https://accessibility.windows.example.org/ns/class"
def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
if platform == "ubuntu":
_state_ns = state_ns_ubuntu
_component_ns = component_ns_ubuntu
elif platform == "windows":
_state_ns = state_ns_windows
_component_ns = component_ns_windows
else:
raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
keeps: bool = node.tag.startswith("document") \
or node.tag.endswith("item") \
or node.tag.endswith("button") \
@@ -53,23 +67,26 @@ def judge_node(node: ET, platform="ubuntu", check_image=False) -> bool:
, "traydummysearchcontrol", "uiimage", "uiproperty"
, "uiribboncommandbar"
}
keeps = keeps and (platform == "ubuntu" \
and node.get("{{{:}}}showing".format(state_ns), "false") == "true" \
and node.get("{{{:}}}visible".format(state_ns), "false") == "true" \
or platform == "windows" \
and node.get("{{{:}}}visible".format(state_ns), "false") == "true" \
) \
and (node.get("{{{:}}}enabled".format(state_ns), "false") == "true" \
or node.get("{{{:}}}editable".format(state_ns), "false") == "true" \
or node.get("{{{:}}}expandable".format(state_ns), "false") == "true" \
or node.get("{{{:}}}checkable".format(state_ns), "false") == "true"
) \
and (node.get("name", "") != "" or node.text is not None and len(node.text) > 0 \
or check_image and node.get("image", "false") == "true"
)
keeps = keeps and (
platform == "ubuntu"
and node.get("{{{:}}}showing".format(_state_ns), "false") == "true"
and node.get("{{{:}}}visible".format(_state_ns), "false") == "true"
or platform == "windows"
and node.get("{{{:}}}visible".format(_state_ns), "false") == "true"
) \
and (
node.get("{{{:}}}enabled".format(_state_ns), "false") == "true"
or node.get("{{{:}}}editable".format(_state_ns), "false") == "true"
or node.get("{{{:}}}expandable".format(_state_ns), "false") == "true"
or node.get("{{{:}}}checkable".format(_state_ns), "false") == "true"
) \
and (
node.get("name", "") != "" or node.text is not None and len(node.text) > 0 \
or check_image and node.get("image", "false") == "true"
)
coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format(component_ns), "(-1, -1)"))
sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format(component_ns), "(-1, -1)"))
coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format(_component_ns), "(-1, -1)"))
sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format(_component_ns), "(-1, -1)"))
keeps = keeps and coordinates[0] >= 0 and coordinates[1] >= 0 and sizes[0] > 0 and sizes[1] > 0
return keeps
@@ -85,7 +102,19 @@ def filter_nodes(root: ET, platform="ubuntu", check_image=False):
return filtered_nodes
def draw_bounding_boxes(nodes, image_file_content, down_sampling_ratio=1.0):
def draw_bounding_boxes(nodes, image_file_content, down_sampling_ratio=1.0, platform="ubuntu"):
if platform == "ubuntu":
_state_ns = state_ns_ubuntu
_component_ns = component_ns_ubuntu
_value_ns = value_ns_ubuntu
elif platform == "windows":
_state_ns = state_ns_windows
_component_ns = component_ns_windows
_value_ns = value_ns_windows
else:
raise ValueError("Invalid platform, must be 'ubuntu' or 'windows'")
# Load the screenshot image
image_stream = io.BytesIO(image_file_content)
image = Image.open(image_stream)
@@ -107,8 +136,8 @@ def draw_bounding_boxes(nodes, image_file_content, down_sampling_ratio=1.0):
# Loop over all the visible nodes and draw their bounding boxes
for _node in nodes:
coords_str = _node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord')
size_str = _node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size')
coords_str = _node.attrib.get('{{{:}}}screencoord'.format(_component_ns))
size_str = _node.attrib.get('{{{:}}}size'.format(_component_ns))
if coords_str and size_str:
try:
@@ -162,19 +191,15 @@ def draw_bounding_boxes(nodes, image_file_content, down_sampling_ratio=1.0):
node_text = (_node.text if '"' not in _node.text \
else '"{:}"'.format(_node.text.replace('"', '""'))
)
elif _node.get("{uri:deskat:uia.windows.microsoft.org}class", "").endswith("EditWrapper") \
and _node.get("{uri:deskat:value.at-spi.gnome.org}value"):
node_text: str = _node.get("{uri:deskat:value.at-spi.gnome.org}value")
elif _node.get("{{{:}}}class".format(class_ns_windows), "").endswith("EditWrapper") \
and _node.get("{{{:}}}value".format(_value_ns)):
node_text = _node.get("{{{:}}}value".format(_value_ns), "")
node_text = (node_text if '"' not in node_text \
else '"{:}"'.format(node_text.replace('"', '""'))
)
else:
node_text = '""'
text_information: str = "{:d}\t{:}\t{:}\t{:}" \
.format(index, _node.tag
, _node.get("name", "")
, node_text
)
text_information: str = "{:d}\t{:}\t{:}\t{:}".format(index, _node.tag, _node.get("name", ""), node_text)
text_informations.append(text_information)
index += 1