ver Jan9th

implemented get_accessibility_tree to convert the accessibility tree
  from pyatspi to an xml format similar to view hierarchy and HTML
havn't merged into server/main.py
tested the AT structure of main window of Thunderbird

desktop_env/evaluators/getters/__init__.py

@@ -1,3 +1,3 @@
 from .file import get_cloud_file, get_vm_file, get_cache_file
-from .misc import get_rule
+from .misc import get_rule, get_desktop_path, get_wallpaper, get_accessibility_tree
 from .vlc import get_vlc_playing_info

desktop_env/evaluators/getters/misc.py

@@ -1,10 +1,20 @@
-from typing import TypeVar
+from typing import TypeVar, Any
+from typing import Dict, List
 
 import platform
 import subprocess
 import ctypes
 import os
 
+import pyatspi
+from pyatspi import Accessible, StateType
+from pyatspi import Component, Document
+from pyatspi import Text as ATText
+from pyatspi import Value as ATValue
+from pyatspi import Action as ATAction
+import lxml.etree
+from lxml.etree import _Element
+
 import logging
 logger = logging.getLogger("desktopenv.getters.misc")
 
@@ -16,7 +26,7 @@ def get_rule(env, config: R) -> R:
     return config["rules"]
 
 
-def get_desktop_path():
+def get_desktop_path(*args):
     username = os.getlogin()  # Get the current username
     if platform.system() == "Windows":
         return os.path.join("C:", "Users", username, "Desktop")
@@ -28,7 +38,7 @@ def get_desktop_path():
         raise Exception("Unsupported operating system")
 
 
-def get_wallpaper():
+def get_wallpaper(*args):
     def get_wallpaper_windows():
         SPI_GETDESKWALLPAPER = 0x73
         MAX_PATH = 260
@@ -64,3 +74,147 @@ def get_wallpaper():
         return get_wallpaper_linux()
     else:
         return "Unsupported OS"
+
+_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
+                        , "attr": "uri:deskat:attributes.at-spi.gnome.org"
+                        , "cp": "uri:deskat:component.at-spi.gnome.org"
+                        , "doc": "uri:deskat:document.at-spi.gnome.org"
+                        , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
+                        , "txt": "uri:deskat:text.at-spi.gnome.org"
+                        , "val": "uri:deskat:value.at-spi.gnome.org"
+                        , "act": "uri:deskat:action.at-spi.gnome.org"
+                        }
+def _create_node(node: Accessible) -> _Element:
+    attribute_dict: Dict[str, Any] = {"name": node.name}
+
+    #  States {{{ # 
+    states: List[StateType] = node.getState().get_states()
+    for st in states:
+        state_name: str = StateType._enum_lookup[st]
+        attribute_dict[ "{{{:}}}{:}"\
+                            .format( _accessibility_ns_map["st"]
+                                   , state_name.split("_", maxsplit=1)[1].lower()
+                                   )
+                      ] = "true"
+    #  }}} States # 
+
+    #  Attributes {{{ # 
+    attributes: List[str] = node.getAttributes()
+    for attrbt in attributes:
+        attribute_name: str
+        attribute_value: str
+        attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
+        attribute_dict[ "{{{:}}}{:}"\
+                            .format( _accessibility_ns_map["attr"]
+                                   , attribute_name
+                                   )
+                      ] = attribute_value
+    #  }}} Attributes # 
+
+    #  Component {{{ # 
+    try:
+        component: Component = node.queryComponent()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN))
+        attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW))
+        attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT))
+        attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize())
+    #  }}} Component # 
+
+    #  Document {{{ # 
+    try:
+        document: Document = node.queryDocument()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale()
+        attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount())
+        attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber())
+        for attrbt in document.getAttributes():
+            attribute_name: str
+            attribute_value: str
+            attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
+            attribute_dict[ "{{{:}}}{:}"\
+                                .format( _accessibility_ns_map["docattr"]
+                                       , attribute_name
+                                       )
+                          ] = attribute_value
+    #  }}} Document # 
+
+    #  Text {{{ # 
+    try:
+        text_obj: ATText = node.queryText()
+    except NotImplementedError:
+        pass
+    else:
+        # only text shown on current screen is available
+        #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
+        text: str = text_obj.getText(0, text_obj.characterCount)
+    #  }}} Text # 
+
+    #  Selection {{{ # 
+    try:
+       node.querySelection()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["selection"] = "true"
+    #  }}} Selection # 
+
+    #  Value {{{ # 
+    try:
+        value: ATValue = node.queryValue()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
+        attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
+        attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
+        attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
+    #  }}} Value # 
+
+    #  Action {{{ # 
+    try:
+        action: ATAction = node.queryAction()
+    except NotImplementedError:
+        pass
+    else:
+        for i in range(action.nActions):
+            action_name: str = action.getName(i).replace(" ", "-")
+            attribute_dict[ "{{{:}}}{:}_desc"\
+                                .format( _accessibility_ns_map["act"]
+                                       , action_name
+                                       )
+                          ] = action.getDescription(i)
+            attribute_dict[ "{{{:}}}{:}_kb"\
+                                .format( _accessibility_ns_map["act"]
+                                       , action_name
+                                       )
+                          ] = action.getKeyBinding(i)
+    #  }}} Action # 
+
+    xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-")
+                                 , attrib=attribute_dict
+                                 , nsmap=_accessibility_ns_map
+                                 )
+    if "text" in locals() and len(text)>0:
+        xml_node.text = text
+    for ch in node:
+        xml_node.append(_create_node(ch))
+    return xml_node
+
+def get_accessibility_tree(*args) -> _Element:
+    desktop: Accessible = pyatspi.Registry.getDesktop(0)
+    desktop_xml: _Element = _create_node(desktop)
+    return desktop_xml
+
+if __name__ == "__main__":
+    import sys
+    with open(sys.argv[1], "w") as f:
+        f.write( lxml.etree.tostring( get_accessibility_tree()
+                                    , encoding="unicode"
+                                    , pretty_print=True
+                                    )
+               )

desktop_env/evaluators/metrics/thunderbird.py

@@ -1,7 +1,13 @@
 #from playwright.sync_api import sync_playwright, Browser
 #from marionette_driver.marionette import Marionette
 #import marionette
-import pyatspi
+#import pyatspi
+
+import lxml.etree
+from lxml.cssselect import CSSSelector
+from lxml.etree import _Element
+
+from typing import List
 
 if __name__ == "__main__":
     #with sync_playwright() as plwr:
@@ -32,3 +38,16 @@ if __name__ == "__main__":
 
     #registry = pyatspi.Registry.get_default()
     #registry
+
+    #xml = "../../任务数据/Thunderbird/vertical-card-view.xml"
+    xml = "../../任务数据/Thunderbird/vertical-table-view.xml"
+    at: _Element = lxml.etree.parse(xml)
+
+    #elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags
+    #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page
+    #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page
+    #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
+    elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
+    print(len(elements))
+    for elm in elements:
+        print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))