Merge remote-tracking branch 'origin/main'

.gitignore

@@ -173,6 +173,8 @@ branch_flag
 branch-config
 *.syncthing.*.tmp
 cache
+version.folder
+at_processing
 
 test.xlsx
 test2.xlsx

desktop_env/server/main.py

@@ -276,12 +276,12 @@ def _create_atspi_node(node: Accessible, depth: int = 0, flag: Optional[str] = N
         # only text shown on current screen is available
         # attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
         text: str = text_obj.getText(0, text_obj.characterCount)
-        if flag=="thunderbird":
+        #if flag=="thunderbird":
-            # appeard in thunderbird (uFFFC), "Object Replacement Character" in
+        # appeard in thunderbird (uFFFC) (not only in thunderbird), "Object
-            # Unicode, "used as placeholder in text for an otherwise
+        # Replacement Character" in Unicode, "used as placeholder in text for
-            # unspecified object; uFFFD is another "Replacement Character",
+        # an otherwise unspecified object; uFFFD is another "Replacement
-            # just in case
+        # Character", just in case
-            text = text.replace("\ufffc", "").replace("\ufffd", "")
+        text = text.replace("\ufffc", "").replace("\ufffd", "")
     #  }}} Text # 
 
     #  Selection {{{ # 

evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json

@@ -57,7 +57,7 @@
     {
       "type": "launch",
       "parameters": {
-        "command": ["nautilus"]
+        "command": ["nautilus", "/home/user/Documents/Finance"]
       }
     }
   ],

evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json

@@ -45,7 +45,7 @@
       "rules": {
         "include": [],
         "exclude": [
-          "ModuleNotFoundError: No module named"
+          "Error:"
         ]
       }
     }

evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json

@@ -1,7 +1,7 @@
 {
   "id": "b337d106-053f-4d37-8da0-7f9c4043a66b",
   "snapshot": "os",
-  "instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the internet for a tutorial on adding line numbers in Vim and setting it as default for my local Vim.",
+  "instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the Internet for a tutorial on adding absolute line numbers in Vim and setting it as default for my local Vim.",
   "source": "authors",
   "config": [
     {
@@ -52,7 +52,7 @@
         "parameters": {
           "files": [
             {
-              "url": "https://drive.usercontent.google.com/download?id=1CyhWjUS2oov4Fzc0VRwTh6LiS2Qu-T_8&export=download&authuser=0&confirm=t&uuid=9d0e2c62-895c-4bb3-a057-30cae60329ed&at=APZUnTVngSwARjYsWSmhSyHAqwID:1709647023362",
+              "url": "https://drive.usercontent.google.com/download?id=1CyhWjUS2oov4Fzc0VRwTh6LiS2Qu-T_8&export=download&authuser=0&confirm=t&uuid=384ea31d-c9ae-4e81-be19-42035c563014&at=APZUnTU9lsYwMLfWb7RIizGr1D7H:1710085473758",
               "path": "eval.sh"
             }
           ]

mm_agents/accessibility_tree_wrap/heuristic_retrieve.py

@@ -25,35 +25,42 @@ def find_leaf_nodes(xlm_file_str):
     return leaf_nodes
 
 
-def filter_nodes(nodes):
+def filter_nodes(nodes, platform="ubuntu"):
     filtered_nodes = []
 
     for node in nodes:
-        if not node.get('{uri:deskat:state.at-spi.gnome.org}visible', None) == 'true':
+        if node.tag.startswith("document")\
-            # Not visible
+                or node.tag.endswith("item")\
-            continue
+                or node.tag.endswith("button")\
-        # Check if the node is a 'panel'
+                or node.tag.endswith("heading")\
-        if node.tag == 'panel':
+                or node.tag.endswith("label")\
-            # Check if the 'panel' represents an interactive element
+                or node.tag.endswith("bar")\
-            # or if it has certain attributes that are of interest.
+                or node.tag.endswith("searchbox")\
-            # Add your conditions here...
+                or node.tag.endswith("textbox")\
-            if node.get('{uri:deskat:state.at-spi.gnome.org}focusable', 'false') == 'true':
+                or node.tag.endswith("link")\
-                filtered_nodes.append(node)
+                or node.tag.endswith("tabelement")\
-        elif node.tag == 'text':
+                or node.tag.endswith("textfield")\
-            continue
+                or node.tag.endswith("textarea")\
-        elif node.get("name") == "" and node.text is None:
+                or node.tag in [ "alert", "canvas", "check-box"
-            continue
+                               , "combo-box", "entry", "icon"
-        else:
+                               , "image", "paragraph"
-            coords = tuple(
+                               , "section", "slider", "static"
-                map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord').strip('()').split(', ')))
+                               , "table-cell", "terminal", "text"
-            if coords[0] < 0 or coords[1] < 0:
+                               , "netuiribbontab", "start", "trayclockwclass"
-                continue
+                               , "traydummysearchcontrol", "uiimage", "uiproperty"
-            size = tuple(
+                               ]:
-                map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size').strip('()').split(', ')))
+            if ( platform=="ubuntu"\
-            if size[0] <= 0 or size[1] <= 0:
+                        and node.get("{{{:}}}showing".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
-                continue
+                        and node.get("{{{:}}}visible".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
-            # Node is not a 'panel', add to the list.
+                     or platform=="windows"\
-            filtered_nodes.append(node)
+                        and node.get("{{{:}}}visible".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
+                      )\
+                    and node.get("{{{:}}}enabled".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
+                    and (node.get("name", "") != "" or node.text is not None and len(node.text)>0):
+                coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format("uri:deskat:component.at-spi.gnome.org")))
+                sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format("uri:deskat:component.at-spi.gnome.org")))
+                if coordinates[0]>0 and coordinates[1]>0 and sizes[0]>0 and sizes[1]>0:
+                    filtered_nodes.append(node)
 
     return filtered_nodes
 
@@ -134,12 +141,14 @@ def print_nodes_with_indent(nodes, indent=0):
 
 
 if __name__ == '__main__':
-    with open('chrome_desktop_example_1.xml', 'r', encoding='utf-8') as f:
+    import json
-        xml_file_str = f.read()
+    with open('2.json', 'r', encoding='utf-8') as f:
+        xml_file_str = json.load(f)["AT"]
     filtered_nodes = filter_nodes(find_leaf_nodes(xml_file_str))
     print(len(filtered_nodes))
-    masks = draw_bounding_boxes(filtered_nodes, 'screenshot.png',
+    masks = draw_bounding_boxes( filtered_nodes, '2.png'
-                                'chrome_desktop_example_1_tagged_remove.png', )
+                               , '2.a.png'
+                               )
 
     # print(masks)
     print(len(masks))