Merge remote-tracking branch 'origin/main'

# Conflicts:
#	desktop_env/evaluators/getters/__init__.py
#	desktop_env/evaluators/metrics/__init__.py
#	requirements.txt

desktop_env/server/main.py

@@ -3,18 +3,29 @@ import os
 import platform
 import subprocess
 from pathlib import Path
-from typing import List
+
+import lxml.etree
+from lxml.etree import _Element
+import pyatspi
+from pyatspi import Accessible, StateType
+from pyatspi import Component, Document
+from pyatspi import Text as ATText
+from pyatspi import Value as ATValue
+from pyatspi import Action as ATAction
+
+from typing import List, Dict
+from typing import Any
 
 import Xlib
 import pyautogui
-import requests
 from PIL import Image
 from Xlib import display, X
+from pyxcursor import Xcursor
+
+import requests
 from flask import Flask, request, jsonify, send_file, abort
 from werkzeug.utils import secure_filename
 
-from pyxcursor import Xcursor
-
 app = Flask(__name__)
 
 pyautogui.PAUSE = 0
@@ -102,6 +113,141 @@ def capture_screen_with_cursor():
 
     return send_file(file_path, mimetype='image/png')
 
+_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
+                        , "attr": "uri:deskat:attributes.at-spi.gnome.org"
+                        , "cp": "uri:deskat:component.at-spi.gnome.org"
+                        , "doc": "uri:deskat:document.at-spi.gnome.org"
+                        , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
+                        , "txt": "uri:deskat:text.at-spi.gnome.org"
+                        , "val": "uri:deskat:value.at-spi.gnome.org"
+                        , "act": "uri:deskat:action.at-spi.gnome.org"
+                        }
+def _create_node(node: Accessible) -> _Element:
+    attribute_dict: Dict[str, Any] = {"name": node.name}
+
+    #  States {{{ # 
+    states: List[StateType] = node.getState().get_states()
+    for st in states:
+        state_name: str = StateType._enum_lookup[st]
+        attribute_dict[ "{{{:}}}{:}"\
+                            .format( _accessibility_ns_map["st"]
+                                   , state_name.split("_", maxsplit=1)[1].lower()
+                                   )
+                      ] = "true"
+    #  }}} States # 
+
+    #  Attributes {{{ # 
+    attributes: List[str] = node.getAttributes()
+    for attrbt in attributes:
+        attribute_name: str
+        attribute_value: str
+        attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
+        attribute_dict[ "{{{:}}}{:}"\
+                            .format( _accessibility_ns_map["attr"]
+                                   , attribute_name
+                                   )
+                      ] = attribute_value
+    #  }}} Attributes # 
+
+    #  Component {{{ # 
+    try:
+        component: Component = node.queryComponent()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN))
+        attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW))
+        attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT))
+        attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize())
+    #  }}} Component # 
+
+    #  Document {{{ # 
+    try:
+        document: Document = node.queryDocument()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale()
+        attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount())
+        attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber())
+        for attrbt in document.getAttributes():
+            attribute_name: str
+            attribute_value: str
+            attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
+            attribute_dict[ "{{{:}}}{:}"\
+                                .format( _accessibility_ns_map["docattr"]
+                                       , attribute_name
+                                       )
+                          ] = attribute_value
+    #  }}} Document # 
+
+    #  Text {{{ # 
+    try:
+        text_obj: ATText = node.queryText()
+    except NotImplementedError:
+        pass
+    else:
+        # only text shown on current screen is available
+        #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
+        text: str = text_obj.getText(0, text_obj.characterCount)
+    #  }}} Text # 
+
+    #  Selection {{{ # 
+    try:
+       node.querySelection()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["selection"] = "true"
+    #  }}} Selection # 
+
+    #  Value {{{ # 
+    try:
+        value: ATValue = node.queryValue()
+    except NotImplementedError:
+        pass
+    else:
+        attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
+        attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
+        attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
+        attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
+    #  }}} Value # 
+
+    #  Action {{{ # 
+    try:
+        action: ATAction = node.queryAction()
+    except NotImplementedError:
+        pass
+    else:
+        for i in range(action.nActions):
+            action_name: str = action.getName(i).replace(" ", "-")
+            attribute_dict[ "{{{:}}}{:}_desc"\
+                                .format( _accessibility_ns_map["act"]
+                                       , action_name
+                                       )
+                          ] = action.getDescription(i)
+            attribute_dict[ "{{{:}}}{:}_kb"\
+                                .format( _accessibility_ns_map["act"]
+                                       , action_name
+                                       )
+                          ] = action.getKeyBinding(i)
+    #  }}} Action # 
+
+    xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-")
+                                 , attrib=attribute_dict
+                                 , nsmap=_accessibility_ns_map
+                                 )
+    if "text" in locals() and len(text)>0:
+        xml_node.text = text
+    for ch in node:
+        xml_node.append(_create_node(ch))
+    return xml_node
+
+@app.route("/accessibility", methods=["GET"])
+def get_accessibility_tree():
+    desktop: Accessible = pyatspi.Registry.getDesktop(0)
+    desktop_xml: _Element = _create_node(desktop)
+    return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
 
 @app.route('/screen_size', methods=['POST'])
 def get_screen_size():