lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-18 23:46:25 +08:00
parent 4af19fb777 21314346c5
commit e4055ab798
1 changed files with 15 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
desktop_env/server/main.py
View File

@@ -140,14 +140,15 @@ def get_terminal_output():
if user_platform == "Linux": if user_platform == "Linux":
desktop: Accessible = pyatspi.Registry.getDesktop(0) desktop: Accessible = pyatspi.Registry.getDesktop(0)
if _has_active_terminal(desktop): if _has_active_terminal(desktop):
desktop_xml: _Element = _create_node(desktop) desktop_xml: _Element = _create_atspi_node(desktop)
# 1. the terminal window (frame of application is st:active) is open and active # 1. the terminal window (frame of application is st:active) is open and active
# 2. the terminal tab (terminal status is st:focused) is focused # 2. the terminal tab (terminal status is st:focused) is focused
xpath = '//application[@name="gnome-terminal-server"]/frame[@st:active="true"]//terminal[@st:focused="true"]' xpath = '//application[@name="gnome-terminal-server"]/frame[@st:active="true"]//terminal[@st:focused="true"]'
terminals: List[_Element] = desktop_xml.xpath(xpath, namespaces=_accessibility_ns_map) terminals: List[_Element] = desktop_xml.xpath(xpath, namespaces=_accessibility_ns_map)
output = terminals[0].text.rstrip() if len(terminals) == 1 else None output = terminals[0].text.rstrip() if len(terminals) == 1 else None
else: # windows and macos platform is not implemented currently else: # windows and macos platform is not implemented currently
raise NotImplementedError #raise NotImplementedError
return "Currently not implemented for platform {:}.".format(platform.platform()), 500
return jsonify({"output": output, "status": "success"}) return jsonify({"output": output, "status": "success"})
except: except:
return jsonify({"output": None, "status": "error"}) return jsonify({"output": None, "status": "error"})
@@ -164,7 +165,7 @@ _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
} }
def _create_node(node: Accessible) -> _Element: def _create_atspi_node(node: Accessible) -> _Element:
attribute_dict: Dict[str, Any] = {"name": node.name} attribute_dict: Dict[str, Any] = {"name": node.name}
# States {{{ # # States {{{ #
@@ -285,15 +286,22 @@ def _create_node(node: Accessible) -> _Element:
if "text" in locals() and len(text) > 0: if "text" in locals() and len(text) > 0:
xml_node.text = text xml_node.text = text
for ch in node: for ch in node:
xml_node.append(_create_node(ch)) xml_node.append(_create_atspi_node(ch))
return xml_node return xml_node
@app.route("/accessibility", methods=["GET"]) @app.route("/accessibility", methods=["GET"])
def get_accessibility_tree(): def get_accessibility_tree():
desktop: Accessible = pyatspi.Registry.getDesktop(0) os_name: str = platform.system()
desktop_xml: _Element = _create_node(desktop)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")}) # AT-SPI works for KDE as well
if os_name == "Linux":
desktop: Accessible = pyatspi.Registry.getDesktop(0)
desktop_xml: _Element = _create_atspi_node(desktop)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
# TODO: Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned
else:
return "Currently not implemented for platform {:}.".format(platform.platform()), 500
@app.route('/screen_size', methods=['POST']) @app.route('/screen_size', methods=['POST'])