lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan26thv3

Browse Source 
fixed bug caused by an empty node.text
remove nodes whose name and text are all empty
This commit is contained in:
David Chang
2024-01-26 23:47:46 +08:00
parent b7e3e1f7d0
commit eef5158663
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
View File

@@ -40,6 +40,8 @@ def filter_nodes(nodes):
filtered_nodes.append(node) filtered_nodes.append(node)
elif node.tag == 'text': elif node.tag == 'text':
continue continue
elif node.get("name") == "" and node.text is None:
continue
else: else:
coords = tuple( coords = tuple(
map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord').strip('()').split(', '))) map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord').strip('()').split(', ')))

7
mm_agents/gpt_4v_agent.py
View File

@@ -39,7 +39,10 @@ def linearize_accessibility_tree(accessibility_tree):
for node in filtered_nodes: for node in filtered_nodes:
linearized_accessibility_tree += node.tag + "\t" linearized_accessibility_tree += node.tag + "\t"
linearized_accessibility_tree += node.attrib.get('name') + "\t" linearized_accessibility_tree += node.attrib.get('name') + "\t"
linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t" if node.text:
linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t"
else:
linearized_accessibility_tree += '""\t'
linearized_accessibility_tree += node.attrib.get( linearized_accessibility_tree += node.attrib.get(
'{uri:deskat:component.at-spi.gnome.org}screencoord') + "\t" '{uri:deskat:component.at-spi.gnome.org}screencoord') + "\t"
linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size') + "\n" linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size') + "\n"
@@ -87,7 +90,7 @@ def parse_actions_from_string(input_string):
try: try:
action_dict = json.loads(input_string) action_dict = json.loads(input_string)
return [action_dict] return [action_dict]
except json.JSONDecodeError as e: except json.JSONDecodeError:
raise ValueError("Invalid response format: " + input_string) raise ValueError("Invalid response format: " + input_string)