From 303c89411eb3c83da78164cd18a0d680499e8a5a Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Tue, 23 Jan 2024 14:37:54 +0800 Subject: [PATCH] Fix bugs of a11y tree when the tag is empty --- desktop_env/server/main.py | 92 ++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index cccb0d4..df31e99 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -14,7 +14,7 @@ import pyautogui import requests from PIL import Image from Xlib import display, X -from flask import Flask, request, jsonify, send_file, abort #, send_from_directory +from flask import Flask, request, jsonify, send_file, abort # , send_from_directory from lxml.etree import _Element from pyatspi import Accessible, StateType from pyatspi import Action as ATAction @@ -33,6 +33,7 @@ logger = app.logger recording_process = None # fixme: this is a temporary solution for recording, need to be changed to support multiple-process recording_path = "/tmp/recording.mp4" + @app.route('/setup/execute', methods=['POST']) @app.route('/execute', methods=['POST']) def execute_command(): @@ -147,7 +148,7 @@ def get_terminal_output(): terminals: List[_Element] = desktop_xml.xpath(xpath, namespaces=_accessibility_ns_map) output = terminals[0].text.rstrip() if len(terminals) == 1 else None else: # windows and macos platform is not implemented currently - #raise NotImplementedError + # raise NotImplementedError return "Currently not implemented for platform {:}.".format(platform.platform()), 500 return jsonify({"output": output, "status": "success"}) except: @@ -172,11 +173,10 @@ def _create_atspi_node(node: Accessible) -> _Element: states: List[StateType] = node.getState().get_states() for st in states: state_name: str = StateType._enum_lookup[st] - attribute_dict["{{{:}}}{:}" \ - .format(_accessibility_ns_map["st"] - , state_name.split("_", maxsplit=1)[1].lower() - ) - ] = "true" + if len(state_name.split("_", maxsplit=1)[1].lower()) == 0: + continue + attribute_dict[ + "{{{:}}}{:}".format(_accessibility_ns_map["st"], state_name.split("_", maxsplit=1)[1].lower())] = "true" # }}} States # # Attributes {{{ # @@ -185,11 +185,9 @@ def _create_atspi_node(node: Accessible) -> _Element: attribute_name: str attribute_value: str attribute_name, attribute_value = attrbt.split(":", maxsplit=1) - attribute_dict["{{{:}}}{:}" \ - .format(_accessibility_ns_map["attr"] - , attribute_name - ) - ] = attribute_value + if len(attribute_name) == 0: + continue + attribute_dict["{{{:}}}{:}".format(_accessibility_ns_map["attr"], attribute_name)] = attribute_value # }}} Attributes # # Component {{{ # @@ -220,11 +218,9 @@ def _create_atspi_node(node: Accessible) -> _Element: attribute_name: str attribute_value: str attribute_name, attribute_value = attrbt.split(":", maxsplit=1) - attribute_dict["{{{:}}}{:}" \ - .format(_accessibility_ns_map["docattr"] - , attribute_name - ) - ] = attribute_value + if len(attribute_name) == 0: + continue + attribute_dict["{{{:}}}{:}".format(_accessibility_ns_map["docattr"], attribute_name)] = attribute_value # }}} Document # # Text {{{ # @@ -277,12 +273,18 @@ def _create_atspi_node(node: Accessible) -> _Element: , action_name ) ] = action.getKeyBinding(i) - # }}} Action # + # }}} Action # - xml_node = lxml.etree.Element(node.getRoleName().replace(" ", "-") - , attrib=attribute_dict - , nsmap=_accessibility_ns_map - ) + if node.getRoleName().strip() == "": + node_role_name = "unknown" + else: + node_role_name = node.getRoleName().replace(" ", "-") + + xml_node = lxml.etree.Element( + node_role_name, + attrib=attribute_dict, + nsmap=_accessibility_ns_map + ) if "text" in locals() and len(text) > 0: xml_node.text = text for ch in node: @@ -589,7 +591,7 @@ def activate_window(): window_name = data.get('window_name', None) if not window_name: return "window_name required", 400 - strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string + strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string by_class_name: bool = data.get("by_class", False) os_name = platform.system() @@ -601,11 +603,11 @@ def activate_window(): windows: List[gw.Window] = gw.getWindowsWithTitle(window_name) window: Optional[gw.Window] = None - if len(windows)==0: + if len(windows) == 0: return "Window {:} not found (empty results)".format(window_name), 404 elif strict: for wnd in windows: - if wnd.title==wnd: + if wnd.title == wnd: window = wnd if window is None: return "Window {:} not found (strict mode).".format(window_name), 404 @@ -621,11 +623,11 @@ def activate_window(): windows = gw.getWindowsWithTitle(window_name) window: Optional[gw.Window] = None - if len(windows)==0: + if len(windows) == 0: return "Window {:} not found (empty results)".format(window_name), 404 elif strict: for wnd in windows: - if wnd.title==wnd: + if wnd.title == wnd: window = wnd if window is None: return "Window {:} not found (strict mode).".format(window_name), 404 @@ -638,26 +640,27 @@ def activate_window(): elif os_name == 'Linux': # Attempt to activate VS Code window using wmctrl - subprocess.run( [ "wmctrl" - , "-{:}{:}a".format( "x" if by_class_name else "" - , "F" if strict else "" - ) - , window_name + subprocess.run(["wmctrl" + , "-{:}{:}a".format("x" if by_class_name else "" + , "F" if strict else "" + ) + , window_name ] - ) + ) else: return f"Operating system {os_name} not supported.", 400 return "Window activated successfully", 200 + @app.route("/setup/close_window", methods=["POST"]) def close_window(): data = request.json if "window_name" not in data: return "window_name required", 400 window_name: str = data["window_name"] - strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string + strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string by_class_name: bool = data.get("by_class", False) os_name: str = platform.system() @@ -669,11 +672,11 @@ def close_window(): windows: List[gw.Window] = gw.getWindowsWithTitle(window_name) window: Optional[gw.Window] = None - if len(windows)==0: + if len(windows) == 0: return "Window {:} not found (empty results)".format(window_name), 404 elif strict: for wnd in windows: - if wnd.title==wnd: + if wnd.title == wnd: window = wnd if window is None: return "Window {:} not found (strict mode).".format(window_name), 404 @@ -681,14 +684,14 @@ def close_window(): window = windows[0] window.close() elif os_name == "Linux": - subprocess.run( [ "wmctrl" - , "-{:}{:}c".format( "x" if by_class_name else "" - , "F" if strict else "" - ) - , window_name + subprocess.run(["wmctrl" + , "-{:}{:}c".format("x" if by_class_name else "" + , "F" if strict else "" + ) + , window_name ] - ) - elif os_name=="Darwin": + ) + elif os_name == "Darwin": import pygetwindow as gw return "Currently not supported on macOS.", 500 else: @@ -696,6 +699,7 @@ def close_window(): return "Window closed successfully.", 200 + @app.route('/start_recording', methods=['POST']) def start_recording(): global recording_process @@ -722,7 +726,7 @@ def end_recording(): recording_process.terminate() recording_process.wait() - #return_code = recording_process.returncode + # return_code = recording_process.returncode output, error = recording_process.communicate() recording_process = None