Merge remote-tracking branch 'origin/main'
# Conflicts: # desktop_env/evaluators/getters/__init__.py # desktop_env/evaluators/metrics/__init__.py # requirements.txt
This commit is contained in:
Timothyxxx
73
desktop_env/server/README.md
Normal file
73
desktop_env/server/README.md
Normal file
@@ -0,0 +1,73 @@
|
||||
<!-- vimc: call SyntaxRange#Include('```xml', '```', 'xml', 'NonText'): -->
|
||||
<!-- vimc: call SyntaxRange#Include('```css', '```', 'css', 'NonText'): -->
|
||||
<!-- vimc: call SyntaxRange#Include('```sh', '```', 'sh', 'NonText'): -->
|
||||
|
||||
### About the Converted Accessibility Tree
|
||||
|
||||
For several applications like Firefox or Thunderbird, you should first enable
|
||||
|
||||
```sh
|
||||
gsettings set org.gnome.desktop.interface toolkit-accessibility true
|
||||
```
|
||||
|
||||
to see their accessibility tree.
|
||||
|
||||
#### Example of AT
|
||||
|
||||
An example of a node:
|
||||
|
||||
```xml
|
||||
<section xmlns:attr="uri:deskat:attributes.at-spi.gnome.org" attr:class="subject" st:enabled="true" cp:screencoord="(1525, 169)", cp:windowcoord="(342, 162)", cp:size="(327, 21)">
|
||||
歡迎使用新的 Outlook.com 帳戶
|
||||
</section>
|
||||
```
|
||||
|
||||
An example of a tree:
|
||||
|
||||
```xml
|
||||
<desktop-frame ...>
|
||||
<application name="Thunderbird" ...>
|
||||
... <!-- nodes of windows -->
|
||||
</application>
|
||||
...
|
||||
</desktop-frame>
|
||||
```
|
||||
|
||||
#### Useful attributes
|
||||
|
||||
1. `name` - shows the name of application, title of window, or name of some
|
||||
component
|
||||
2. `attr:class` - somewhat the same role as `class` in HTML
|
||||
3. `attr:id` - somewhat the same role as `id` in HTML
|
||||
4. `cp:screencoord` - absolute coordinator on the screen
|
||||
5. `cp:windowcoord` - relative coordinator in the window
|
||||
6. `cp:size` - the size
|
||||
|
||||
Also several states like `st:enabled` and `st:visible` can be indicated. A full
|
||||
state list is available at
|
||||
<https://gitlab.gnome.org/GNOME/pyatspi2/-/blob/master/pyatspi/state.py?ref_type=heads>.
|
||||
|
||||
#### How to use it in evaluation
|
||||
|
||||
See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and
|
||||
function `check_accessibility_tree` in `metrics/general.py`. You can use CSS
|
||||
selector or XPath to reference a target nodes. You can also check its text
|
||||
contents.
|
||||
|
||||
An example of a CSS selector:
|
||||
|
||||
```css
|
||||
application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]
|
||||
```
|
||||
|
||||
This selector will select the page tab of profile manager in Thunderbird (if open).
|
||||
|
||||
For usage of CSS selector: <https://www.w3.org/TR/selectors-3/>. For usage of XPath: <https://www.w3.org/TR/xpath-31/>.
|
||||
|
||||
#### Manual check
|
||||
|
||||
You can use accerciser to check the accessibility tree on GNOME VM.
|
||||
|
||||
```sh
|
||||
sudo apt install accerciser
|
||||
```
|
||||
@@ -3,18 +3,29 @@ import os
|
||||
import platform
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import lxml.etree
|
||||
from lxml.etree import _Element
|
||||
import pyatspi
|
||||
from pyatspi import Accessible, StateType
|
||||
from pyatspi import Component, Document
|
||||
from pyatspi import Text as ATText
|
||||
from pyatspi import Value as ATValue
|
||||
from pyatspi import Action as ATAction
|
||||
|
||||
from typing import List, Dict
|
||||
from typing import Any
|
||||
|
||||
import Xlib
|
||||
import pyautogui
|
||||
import requests
|
||||
from PIL import Image
|
||||
from Xlib import display, X
|
||||
from pyxcursor import Xcursor
|
||||
|
||||
import requests
|
||||
from flask import Flask, request, jsonify, send_file, abort
|
||||
from werkzeug.utils import secure_filename
|
||||
|
||||
from pyxcursor import Xcursor
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
pyautogui.PAUSE = 0
|
||||
@@ -102,6 +113,141 @@ def capture_screen_with_cursor():
|
||||
|
||||
return send_file(file_path, mimetype='image/png')
|
||||
|
||||
_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
|
||||
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
|
||||
, "cp": "uri:deskat:component.at-spi.gnome.org"
|
||||
, "doc": "uri:deskat:document.at-spi.gnome.org"
|
||||
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
|
||||
, "txt": "uri:deskat:text.at-spi.gnome.org"
|
||||
, "val": "uri:deskat:value.at-spi.gnome.org"
|
||||
, "act": "uri:deskat:action.at-spi.gnome.org"
|
||||
}
|
||||
def _create_node(node: Accessible) -> _Element:
|
||||
attribute_dict: Dict[str, Any] = {"name": node.name}
|
||||
|
||||
# States {{{ #
|
||||
states: List[StateType] = node.getState().get_states()
|
||||
for st in states:
|
||||
state_name: str = StateType._enum_lookup[st]
|
||||
attribute_dict[ "{{{:}}}{:}"\
|
||||
.format( _accessibility_ns_map["st"]
|
||||
, state_name.split("_", maxsplit=1)[1].lower()
|
||||
)
|
||||
] = "true"
|
||||
# }}} States #
|
||||
|
||||
# Attributes {{{ #
|
||||
attributes: List[str] = node.getAttributes()
|
||||
for attrbt in attributes:
|
||||
attribute_name: str
|
||||
attribute_value: str
|
||||
attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
|
||||
attribute_dict[ "{{{:}}}{:}"\
|
||||
.format( _accessibility_ns_map["attr"]
|
||||
, attribute_name
|
||||
)
|
||||
] = attribute_value
|
||||
# }}} Attributes #
|
||||
|
||||
# Component {{{ #
|
||||
try:
|
||||
component: Component = node.queryComponent()
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN))
|
||||
attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW))
|
||||
attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT))
|
||||
attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize())
|
||||
# }}} Component #
|
||||
|
||||
# Document {{{ #
|
||||
try:
|
||||
document: Document = node.queryDocument()
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale()
|
||||
attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount())
|
||||
attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber())
|
||||
for attrbt in document.getAttributes():
|
||||
attribute_name: str
|
||||
attribute_value: str
|
||||
attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
|
||||
attribute_dict[ "{{{:}}}{:}"\
|
||||
.format( _accessibility_ns_map["docattr"]
|
||||
, attribute_name
|
||||
)
|
||||
] = attribute_value
|
||||
# }}} Document #
|
||||
|
||||
# Text {{{ #
|
||||
try:
|
||||
text_obj: ATText = node.queryText()
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
# only text shown on current screen is available
|
||||
#attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
|
||||
text: str = text_obj.getText(0, text_obj.characterCount)
|
||||
# }}} Text #
|
||||
|
||||
# Selection {{{ #
|
||||
try:
|
||||
node.querySelection()
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
attribute_dict["selection"] = "true"
|
||||
# }}} Selection #
|
||||
|
||||
# Value {{{ #
|
||||
try:
|
||||
value: ATValue = node.queryValue()
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
|
||||
attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
|
||||
attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
|
||||
attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
|
||||
# }}} Value #
|
||||
|
||||
# Action {{{ #
|
||||
try:
|
||||
action: ATAction = node.queryAction()
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
for i in range(action.nActions):
|
||||
action_name: str = action.getName(i).replace(" ", "-")
|
||||
attribute_dict[ "{{{:}}}{:}_desc"\
|
||||
.format( _accessibility_ns_map["act"]
|
||||
, action_name
|
||||
)
|
||||
] = action.getDescription(i)
|
||||
attribute_dict[ "{{{:}}}{:}_kb"\
|
||||
.format( _accessibility_ns_map["act"]
|
||||
, action_name
|
||||
)
|
||||
] = action.getKeyBinding(i)
|
||||
# }}} Action #
|
||||
|
||||
xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-")
|
||||
, attrib=attribute_dict
|
||||
, nsmap=_accessibility_ns_map
|
||||
)
|
||||
if "text" in locals() and len(text)>0:
|
||||
xml_node.text = text
|
||||
for ch in node:
|
||||
xml_node.append(_create_node(ch))
|
||||
return xml_node
|
||||
|
||||
@app.route("/accessibility", methods=["GET"])
|
||||
def get_accessibility_tree():
|
||||
desktop: Accessible = pyatspi.Registry.getDesktop(0)
|
||||
desktop_xml: _Element = _create_node(desktop)
|
||||
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
|
||||
|
||||
@app.route('/screen_size', methods=['POST'])
|
||||
def get_screen_size():
|
||||
|
||||
Reference in New Issue
Block a user