lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish loading the vscode examples v1; Improve on the infra: Add accessibility tree into the observation; Add activate window function, etc

Browse Source
This commit is contained in:
Timothyxxx
2024-01-14 18:30:49 +08:00
parent 2228f346a9
commit d52b692ee5
16 changed files with 368 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
desktop_env/server/README.md
View File

@@ -71,3 +71,10 @@ You can use accerciser to check the accessibility tree on GNOME VM.
```sh
sudo apt install accerciser
```
### Additional Installation
Activating the window manager control requires the installation of `wmctrl`:
```bash
sudo apt install wmctrl
```

158
desktop_env/server/main.py
View File

@@ -3,29 +3,26 @@ import os
import platform
import subprocess
from pathlib import Path
from typing import Any, Optional
from typing import List, Dict
import Xlib
import lxml.etree
from lxml.etree import _Element
import pyatspi
import pyautogui
import requests
from PIL import Image
from Xlib import display, X
from flask import Flask, request, jsonify, send_file, abort
from lxml.etree import _Element
from pyatspi import Accessible, StateType
from pyatspi import Action as ATAction
from pyatspi import Component, Document
from pyatspi import Text as ATText
from pyatspi import Value as ATValue
from pyatspi import Action as ATAction
from typing import List, Dict
from typing import Any, Optional
import Xlib
import pyautogui
from PIL import Image
from Xlib import display, X
from pyxcursor import Xcursor
import requests
from flask import Flask, request, jsonify, send_file, abort
from werkzeug.utils import secure_filename
app = Flask(__name__)
pyautogui.PAUSE = 0
@@ -140,22 +137,24 @@ def get_terminal_output():
xpath = '//application[@name="gnome-terminal-server"]/frame[@st:active="true"]//terminal[@st:focused="true"]'
terminals: List[_Element] = desktop_xml.xpath(xpath, namespaces=_accessibility_ns_map)
output = terminals[0].text.rstrip() if len(terminals) == 1 else None
else: # windows and macos platform is not implemented currently
else: # windows and macos platform is not implemented currently
raise NotImplementedError
return jsonify({"output": output, "status": "success"})
except:
return jsonify({"output": None, "status": "error"})
_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
def _create_node(node: Accessible) -> _Element:
attribute_dict: Dict[str, Any] = {"name": node.name}
@@ -163,11 +162,11 @@ def _create_node(node: Accessible) -> _Element:
states: List[StateType] = node.getState().get_states()
for st in states:
state_name: str = StateType._enum_lookup[st]
attribute_dict[ "{{{:}}}{:}"\
.format( _accessibility_ns_map["st"]
, state_name.split("_", maxsplit=1)[1].lower()
)
] = "true"
attribute_dict["{{{:}}}{:}" \
.format(_accessibility_ns_map["st"]
, state_name.split("_", maxsplit=1)[1].lower()
)
] = "true"
# }}} States #
# Attributes {{{ #
@@ -176,11 +175,11 @@ def _create_node(node: Accessible) -> _Element:
attribute_name: str
attribute_value: str
attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
attribute_dict[ "{{{:}}}{:}"\
.format( _accessibility_ns_map["attr"]
, attribute_name
)
] = attribute_value
attribute_dict["{{{:}}}{:}" \
.format(_accessibility_ns_map["attr"]
, attribute_name
)
] = attribute_value
# }}} Attributes #
# Component {{{ #
@@ -189,9 +188,12 @@ def _create_node(node: Accessible) -> _Element:
except NotImplementedError:
pass
else:
attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN))
attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW))
attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT))
attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(
component.getPosition(pyatspi.XY_SCREEN))
attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(
component.getPosition(pyatspi.XY_WINDOW))
attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(
component.getPosition(pyatspi.XY_PARENT))
attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize())
# }}} Component #
@@ -208,11 +210,11 @@ def _create_node(node: Accessible) -> _Element:
attribute_name: str
attribute_value: str
attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
attribute_dict[ "{{{:}}}{:}"\
.format( _accessibility_ns_map["docattr"]
, attribute_name
)
] = attribute_value
attribute_dict["{{{:}}}{:}" \
.format(_accessibility_ns_map["docattr"]
, attribute_name
)
] = attribute_value
# }}} Document #
# Text {{{ #
@@ -222,13 +224,13 @@ def _create_node(node: Accessible) -> _Element:
pass
else:
# only text shown on current screen is available
#attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
# attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
text: str = text_obj.getText(0, text_obj.characterCount)
# }}} Text #
# Selection {{{ #
try:
node.querySelection()
node.querySelection()
except NotImplementedError:
pass
else:
@@ -255,34 +257,36 @@ def _create_node(node: Accessible) -> _Element:
else:
for i in range(action.nActions):
action_name: str = action.getName(i).replace(" ", "-")
attribute_dict[ "{{{:}}}{:}_desc"\
.format( _accessibility_ns_map["act"]
, action_name
)
] = action.getDescription(i)
attribute_dict[ "{{{:}}}{:}_kb"\
.format( _accessibility_ns_map["act"]
, action_name
)
] = action.getKeyBinding(i)
attribute_dict["{{{:}}}{:}_desc" \
.format(_accessibility_ns_map["act"]
, action_name
)
] = action.getDescription(i)
attribute_dict["{{{:}}}{:}_kb" \
.format(_accessibility_ns_map["act"]
, action_name
)
] = action.getKeyBinding(i)
# }}} Action #
xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-")
, attrib=attribute_dict
, nsmap=_accessibility_ns_map
)
if "text" in locals() and len(text)>0:
xml_node = lxml.etree.Element(node.getRoleName().replace(" ", "-")
, attrib=attribute_dict
, nsmap=_accessibility_ns_map
)
if "text" in locals() and len(text) > 0:
xml_node.text = text
for ch in node:
xml_node.append(_create_node(ch))
return xml_node
@app.route("/accessibility", methods=["GET"])
def get_accessibility_tree():
desktop: Accessible = pyatspi.Registry.getDesktop(0)
desktop_xml: _Element = _create_node(desktop)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
@app.route('/screen_size', methods=['POST'])
def get_screen_size():
d = display.Display()
@@ -562,5 +566,43 @@ def open_file():
return f"Failed to open {path}. Error: {e}", 500
@app.route("/setup/activate_window", methods=['POST'])
def activate_window():
data = request.json
window_name = data.get('window_name', None)
os_name = platform.system()
if os_name == 'Windows':
import pygetwindow as gw
try:
# Find the VS Code window
vscode_window = gw.getWindowsWithTitle(window_name)[0]
# Activate the window, bringing it to the front
vscode_window.activate()
except IndexError:
return "VS Code window not found.", 404
elif os_name == 'Darwin':
import pygetwindow as gw
try:
# Find the VS Code window
vscode_window = gw.getWindowsWithTitle(window_name)[0]
# Un-minimize the window and then bring it to the front
vscode_window.unminimize()
vscode_window.activate()
except IndexError:
return "VS Code window not found.", 404
elif os_name == 'Linux':
# Attempt to activate VS Code window using wmctrl
subprocess.Popen(["wmctrl", "-a", window_name])
else:
return f"Operating system {os_name} not supported.", 400
return "File opened successfully", 200
if __name__ == '__main__':
app.run(debug=True, host="0.0.0.0")