From ec3bc3079fb0b79a73c111252f820b9d3511a6f5 Mon Sep 17 00:00:00 2001 From: David Chang Date: Tue, 9 Jan 2024 23:01:23 +0800 Subject: [PATCH] ver Jan9th implemented get_accessibility_tree to convert the accessibility tree from pyatspi to an xml format similar to view hierarchy and HTML havn't merged into server/main.py tested the AT structure of main window of Thunderbird --- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/getters/misc.py | 160 +++++++++++++++++- desktop_env/evaluators/metrics/thunderbird.py | 21 ++- 3 files changed, 178 insertions(+), 5 deletions(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 1fd3c02..ed68081 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,3 +1,3 @@ from .file import get_cloud_file, get_vm_file, get_cache_file -from .misc import get_rule +from .misc import get_rule, get_desktop_path, get_wallpaper, get_accessibility_tree from .vlc import get_vlc_playing_info diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index f66322b..51c0732 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -1,10 +1,20 @@ -from typing import TypeVar +from typing import TypeVar, Any +from typing import Dict, List import platform import subprocess import ctypes import os +import pyatspi +from pyatspi import Accessible, StateType +from pyatspi import Component, Document +from pyatspi import Text as ATText +from pyatspi import Value as ATValue +from pyatspi import Action as ATAction +import lxml.etree +from lxml.etree import _Element + import logging logger = logging.getLogger("desktopenv.getters.misc") @@ -16,7 +26,7 @@ def get_rule(env, config: R) -> R: return config["rules"] -def get_desktop_path(): +def get_desktop_path(*args): username = os.getlogin() # Get the current username if platform.system() == "Windows": return os.path.join("C:", "Users", username, "Desktop") @@ -28,7 +38,7 @@ def get_desktop_path(): raise Exception("Unsupported operating system") -def get_wallpaper(): +def get_wallpaper(*args): def get_wallpaper_windows(): SPI_GETDESKWALLPAPER = 0x73 MAX_PATH = 260 @@ -64,3 +74,147 @@ def get_wallpaper(): return get_wallpaper_linux() else: return "Unsupported OS" + +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } +def _create_node(node: Accessible) -> _Element: + attribute_dict: Dict[str, Any] = {"name": node.name} + + # States {{{ # + states: List[StateType] = node.getState().get_states() + for st in states: + state_name: str = StateType._enum_lookup[st] + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["st"] + , state_name.split("_", maxsplit=1)[1].lower() + ) + ] = "true" + # }}} States # + + # Attributes {{{ # + attributes: List[str] = node.getAttributes() + for attrbt in attributes: + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["attr"] + , attribute_name + ) + ] = attribute_value + # }}} Attributes # + + # Component {{{ # + try: + component: Component = node.queryComponent() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN)) + attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW)) + attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT)) + attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize()) + # }}} Component # + + # Document {{{ # + try: + document: Document = node.queryDocument() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale() + attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount()) + attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber()) + for attrbt in document.getAttributes(): + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["docattr"] + , attribute_name + ) + ] = attribute_value + # }}} Document # + + # Text {{{ # + try: + text_obj: ATText = node.queryText() + except NotImplementedError: + pass + else: + # only text shown on current screen is available + #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount) + text: str = text_obj.getText(0, text_obj.characterCount) + # }}} Text # + + # Selection {{{ # + try: + node.querySelection() + except NotImplementedError: + pass + else: + attribute_dict["selection"] = "true" + # }}} Selection # + + # Value {{{ # + try: + value: ATValue = node.queryValue() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue) + attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue) + attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue) + attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement) + # }}} Value # + + # Action {{{ # + try: + action: ATAction = node.queryAction() + except NotImplementedError: + pass + else: + for i in range(action.nActions): + action_name: str = action.getName(i).replace(" ", "-") + attribute_dict[ "{{{:}}}{:}_desc"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getDescription(i) + attribute_dict[ "{{{:}}}{:}_kb"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getKeyBinding(i) + # }}} Action # + + xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-") + , attrib=attribute_dict + , nsmap=_accessibility_ns_map + ) + if "text" in locals() and len(text)>0: + xml_node.text = text + for ch in node: + xml_node.append(_create_node(ch)) + return xml_node + +def get_accessibility_tree(*args) -> _Element: + desktop: Accessible = pyatspi.Registry.getDesktop(0) + desktop_xml: _Element = _create_node(desktop) + return desktop_xml + +if __name__ == "__main__": + import sys + with open(sys.argv[1], "w") as f: + f.write( lxml.etree.tostring( get_accessibility_tree() + , encoding="unicode" + , pretty_print=True + ) + ) diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py index 162a8ae..581b8d1 100644 --- a/desktop_env/evaluators/metrics/thunderbird.py +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -1,7 +1,13 @@ #from playwright.sync_api import sync_playwright, Browser #from marionette_driver.marionette import Marionette #import marionette -import pyatspi +#import pyatspi + +import lxml.etree +from lxml.cssselect import CSSSelector +from lxml.etree import _Element + +from typing import List if __name__ == "__main__": #with sync_playwright() as plwr: @@ -32,3 +38,16 @@ if __name__ == "__main__": #registry = pyatspi.Registry.get_default() #registry + + #xml = "../../任务数据/Thunderbird/vertical-card-view.xml" + xml = "../../任务数据/Thunderbird/vertical-table-view.xml" + at: _Element = lxml.etree.parse(xml) + + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + print(len(elements)) + for elm in elements: + print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))