From f08fa4912cf8fd6a09a6f5cf018da64f757f5c19 Mon Sep 17 00:00:00 2001
From: David Chang <zdy004007@126.com>
Date: Sun, 10 Mar 2024 18:03:02 +0800
Subject: [PATCH 1/2] ver Mar10th

changed AT element filtering
---
 .gitignore                                    |   1 +
 at_processing/.gitignore                      |   4 +
 at_processing/filter.anal                     | 249 ++++++++++++++++++
 at_processing/ubuntu_node_statistics.py       |  22 ++
 desktop_env/server/main.py                    |  12 +-
 .../415ef462-bed3-493a-ac36-ca8c6d23bf1b.json |   2 +-
 .../heuristic_retrieve.py                     |  69 ++---
 7 files changed, 322 insertions(+), 37 deletions(-)
 create mode 100644 at_processing/.gitignore
 create mode 100644 at_processing/filter.anal
 create mode 100644 at_processing/ubuntu_node_statistics.py

diff --git a/.gitignore b/.gitignore
index fc9dc79..a3543d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -171,6 +171,7 @@ tags-opts
 snapshots
 *.syncthing.*.tmp
 cache
+version.folder
 
 test.xlsx
 test2.xlsx
diff --git a/at_processing/.gitignore b/at_processing/.gitignore
new file mode 100644
index 0000000..1719b90
--- /dev/null
+++ b/at_processing/.gitignore
@@ -0,0 +1,4 @@
+/*.json
+/*.xml
+/*.list
+/*.png
diff --git a/at_processing/filter.anal b/at_processing/filter.anal
new file mode 100644
index 0000000..6c2e203
--- /dev/null
+++ b/at_processing/filter.anal
@@ -0,0 +1,249 @@
+vimc: hi AnalSecTitle term=bold cterm=bold:
+vimc: syn match AnalSecTitle /^|\d\+\.\ .*$/:
+vimc: syn match Comment /^#.*$/:
+vimc: hi AnalMetaInfo term=italic ctermfg=red:
+vimc: syn match AnalMetaInfo /^[A-Z]\:\ /:
+vimc: syn match Todo /\<TODO\>\|^I\:/:
+vimc: hi VIMC ctermfg=grey:
+vimc: syn match VIMC /^vimc\:\ .*\:$/:
+vimc: hi AnalComplete term=reverse,bold ctermfg=green:
+vimc: syn match AnalComplete /✓$/:
+vimc: hi AnalFail term=reverse,bold ctermfg=red:
+vimc: syn match AnalFail /✗$/:
+
+vimc: hi AnalInfoElm term=reverse,bold ctermfg=cyan:
+vimc: syn match AnalInfoElm /\<i$/:
+
+# 分析如何按照UFO的规则过滤AT元素
+
+# E: 列出元素类型
+# P: 列出属性
+# R: 引用
+
+UFO中保留的十一种元素类型：
+E: Button, Edit, TabItem, Document, ListItem, MenuItem, ScrollBar, TreeItem, Hyperlink, ComboBox, RadioButton
+
+|1. GNOME上元素类型
+
+R: ubuntu-tag.shrunk.list
+alert 3 ✓
+animation 29
+application 12
+canvas 6 ✓
+check-box 17 ✓
+check-menu-item 1263 ✓
+combo-box 19 ✓
+desktop-frame 1
+dialog 1
+document-frame 4 ✓
+document-presentation 1 ✓
+document-spreadsheet 1 ✓
+document-text 1 ✓
+document-web 10 ✓
+drawing-area 1
+embedded 1
+entry 8 ✓
+filler 264
+form 2
+frame 15
+heading 36 ✓
+icon 96 ✓
+image 246 ✓
+image-map 1
+internal-frame 10
+label 170 ✓
+landmark 12
+layered-pane 3
+link 315 ✓
+list 17
+list-box 7
+list-item 145 ✓
+menu 546
+menu-bar 12
+menu-item 2392 ✓
+notification 45
+page 21
+page-tab 16
+page-tab-list 9
+panel 362
+paragraph 42 ✓
+popup-menu 9
+push-button 612 ✓
+radio-button 28 ✓
+radio-menu-item 174 ✓
+root-pane 3
+ruler 1
+scroll-bar 46 ✓
+scroll-pane 28
+section 568 ✓
+separator 879
+slider 1 ✓
+split-pane 11
+static 76 ✓
+status-bar 7
+table 5
+table-cell 841 ✓
+table-column-header 42 ✓
+table-row 8
+terminal 1 ✓
+text 27 ✓
+toggle-button 123 ✓
+tool-bar 63
+tree 5
+tree-item 14 ✓
+tree-table 1
+unknown 9
+viewport 13
+
+| 2. Windows上元素类型
+
+R: windows-tag.list
+_wwf 1
+address-band-root 1
+backforwardbutton 1 ✓
+boxlayoutview 1
+breadcrumb-parent 1
+browserappmenubutton 1 ✓
+browsercaptionbuttoncontainer 1
+browserframeviewwin 1
+browserrootview 1
+browserview 1
+bubbledialogdelegateview 1
+bubbleframeview 1
+button 6 ✓
+cabinetwclass 1
+chrome_widgetwin_1 2
+consolewindowclass 1
+container 1
+desktop 1
+dialogclientview 1
+duilistview 1
+duiviewwndclassname 1
+dynamiccontent1 1
+dynamiccontent2 1
+edgeavatartoolbarbutton 1 ✓
+edgehubappstowerscreenshotbutton 1 ✓
+edgehubappstowerscrollview 1
+edgehubappstowertoggletoolbarbutton 10 ✓
+edgehubappstowerview 1
+edgenewtabbutton 1 ✓
+edgetab 2
+edgetabactionsmenubutton 1 ✓
+edgetabclosebutton 2 ✓
+edgetabcontainerimpl 1
+edgetabstrip 1
+edgetabstripregionview 1
+element 1
+excel- 2
+excel2 2
+framegrabhandle 1
+fullpageuihost 3
+hubtoggletoolbarbutton 3 ✓
+imemodebutton 1 ✓
+linevscrollbar 1 ✓
+locationbarview 1
+locationiconview 1
+mdiclient 1
+metadatalabel 1 ✓
+modernsearchbox 1 ✓
+msctls_progress32 1
+mstasklistwclass 1
+netuiappframehelper 12
+netuielement 3
+netuifullpageuiwindow 3
+netuihwnd 1
+netuihyperlink 3 ✓
+netuikeyboardtabelement 3 ✓
+netuilabel 9 ✓
+netuilistview 6
+netuilistviewitem 27 ✓
+netuiofficecaption 3
+netuioutspacebutton 3 ✓
+netuipanviewer 3
+netuiribbontab 15 ✓
+netuiscrollbar 6 ✓
+netuiscrollviewer 12
+netuisimplebutton 6 ✓
+netuislabcontainer 6
+netuislabexpandcollapsebutton 3 ✓
+netuitabheader 6 ✓
+nonclientview 2
+nuipane 1
+omniboxviewviews 1
+opusapp 1
+pageactioniconcontainerview 1
+pageactioniconview 1
+pptframeclass 1
+progman 1
+propertreehost 1
+rebarwindow32 1
+reloadbutton 1 ✓
+rootview 1
+scrollview 1
+scrollview--viewport 1
+selectorbutton 2 ✓
+selectornodefault 1
+separator 2
+separatorband 2
+shell_traywnd 1
+shelltabwindowclass 1
+sidebarcontentssplitview 5
+splitwindowcontainerview 1
+start 1 ✓
+starview 1
+statusbarmoduleinner 1
+syslistview32 1
+syspager 1
+systreeview32 1
+tabstrip--edgetabdragcontextimpl 1
+textfield 1 ✓
+toolbarhubiconcontainerview 1
+toolbarsplitwindowbutton 1 ✓
+toolbarview 1
+toolbarview--containerview 1
+toolbarwindow32 5
+topcontaineroverlayview 1
+topcontainerview 1
+towerhubiconcontainerview 1
+travelband 1
+traybutton 2 ✓
+trayclockwclass 1 ✓
+traydummysearchcontrol 1 ✓
+trayinputindicatorwclass 1
+traynotifywnd 1
+trayshowdesktopbuttonwclass 1
+uicolumnheader 4
+uiimage 24 ✓
+uiitem 24 ✓
+uiitemsview 1
+uiproperty 96 ✓
+uiribboncommandbar 1 ✓
+uiribboncommandbardock 1
+uiribbonworkpane 1
+uiviewheader 1 ✓
+universalsearchband 1
+unknown 92
+upband 1
+view 7
+windowedfindbarfocusproxyview 1
+windowscaptionbutton 3 ✓
+worker-window 1
+xldesk 1
+xlmain 1
+
+|3. 筛选属性
+
+UFO中筛选的几种元素属性：
+P: is_visible, is_enabled, title_list, class_name_list
+
+对Ubuntu，照葫芦画瓢，筛选
+P: visible & showing
+P: enabled
+P: name
+P: text
+
+对Windows，筛选
+P: visible
+P: enabled
+P: name
+P: text
diff --git a/at_processing/ubuntu_node_statistics.py b/at_processing/ubuntu_node_statistics.py
new file mode 100644
index 0000000..d00c5e1
--- /dev/null
+++ b/at_processing/ubuntu_node_statistics.py
@@ -0,0 +1,22 @@
+#!/usr/bin/python3
+
+import lxml.etree
+from lxml.etree import _Element
+from typing import Counter
+import json
+import collections
+
+file_name = "w0"
+
+with open("{:}.json".format(file_name)) as f:
+    xml_str: str = json.load(f)["AT"]
+root: _Element = lxml.etree.fromstring(xml_str)
+with open("{:}.xml".format(file_name), "w") as f:
+    f.write(lxml.etree.tostring(root, encoding="unicode", pretty_print=True))
+#root: _Element = lxml.etree.parse("{:}.xml".format(file_name)).getroot()
+node_types: Counter[str] = collections.Counter()
+for n in root.iter():
+    node_types[n.tag] += 1
+
+for n in sorted(node_types):
+    print(n, node_types[n])
diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py
index 4d9763b..f691724 100644
--- a/desktop_env/server/main.py
+++ b/desktop_env/server/main.py
@@ -276,12 +276,12 @@ def _create_atspi_node(node: Accessible, depth: int = 0, flag: Optional[str] = N
         # only text shown on current screen is available
         # attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
         text: str = text_obj.getText(0, text_obj.characterCount)
-        if flag=="thunderbird":
-            # appeard in thunderbird (uFFFC), "Object Replacement Character" in
-            # Unicode, "used as placeholder in text for an otherwise
-            # unspecified object; uFFFD is another "Replacement Character",
-            # just in case
-            text = text.replace("\ufffc", "").replace("\ufffd", "")
+        #if flag=="thunderbird":
+        # appeard in thunderbird (uFFFC) (not only in thunderbird), "Object
+        # Replacement Character" in Unicode, "used as placeholder in text for
+        # an otherwise unspecified object; uFFFD is another "Replacement
+        # Character", just in case
+        text = text.replace("\ufffc", "").replace("\ufffd", "")
     #  }}} Text # 
 
     #  Selection {{{ # 
diff --git a/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json b/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json
index be80232..34d0353 100644
--- a/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json
+++ b/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json
@@ -57,7 +57,7 @@
     {
       "type": "launch",
       "parameters": {
-        "command": ["nautilus"]
+        "command": ["nautilus", "/home/user/Documents/Finance"]
       }
     }
   ],
diff --git a/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py b/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
index 7ab439f..7c45ec3 100644
--- a/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
+++ b/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py
@@ -25,35 +25,42 @@ def find_leaf_nodes(xlm_file_str):
     return leaf_nodes
 
 
-def filter_nodes(nodes):
+def filter_nodes(nodes, platform="ubuntu"):
     filtered_nodes = []
 
     for node in nodes:
-        if not node.get('{uri:deskat:state.at-spi.gnome.org}visible', None) == 'true':
-            # Not visible
-            continue
-        # Check if the node is a 'panel'
-        if node.tag == 'panel':
-            # Check if the 'panel' represents an interactive element
-            # or if it has certain attributes that are of interest.
-            # Add your conditions here...
-            if node.get('{uri:deskat:state.at-spi.gnome.org}focusable', 'false') == 'true':
-                filtered_nodes.append(node)
-        elif node.tag == 'text':
-            continue
-        elif node.get("name") == "" and node.text is None:
-            continue
-        else:
-            coords = tuple(
-                map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord').strip('()').split(', ')))
-            if coords[0] < 0 or coords[1] < 0:
-                continue
-            size = tuple(
-                map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size').strip('()').split(', ')))
-            if size[0] <= 0 or size[1] <= 0:
-                continue
-            # Node is not a 'panel', add to the list.
-            filtered_nodes.append(node)
+        if node.tag.startswith("document")\
+                or node.tag.endswith("item")\
+                or node.tag.endswith("button")\
+                or node.tag.endswith("heading")\
+                or node.tag.endswith("label")\
+                or node.tag.endswith("bar")\
+                or node.tag.endswith("searchbox")\
+                or node.tag.endswith("textbox")\
+                or node.tag.endswith("link")\
+                or node.tag.endswith("tabelement")\
+                or node.tag.endswith("textfield")\
+                or node.tag.endswith("textarea")\
+                or node.tag in [ "alert", "canvas", "check-box"
+                               , "combo-box", "entry", "icon"
+                               , "image", "paragraph"
+                               , "section", "slider", "static"
+                               , "table-cell", "terminal", "text"
+                               , "netuiribbontab", "start", "trayclockwclass"
+                               , "traydummysearchcontrol", "uiimage", "uiproperty"
+                               ]:
+            if ( platform=="ubuntu"\
+                        and node.get("{{{:}}}showing".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
+                        and node.get("{{{:}}}visible".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
+                     or platform=="windows"\
+                        and node.get("{{{:}}}visible".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
+                      )\
+                    and node.get("{{{:}}}enabled".format("uri:deskat:state.at-spi.gnome.org"), "false")=="true"\
+                    and (node.get("name", "") != "" or node.text is not None and len(node.text)>0):
+                coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format("uri:deskat:component.at-spi.gnome.org")))
+                sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format("uri:deskat:component.at-spi.gnome.org")))
+                if coordinates[0]>0 and coordinates[1]>0 and sizes[0]>0 and sizes[1]>0:
+                    filtered_nodes.append(node)
 
     return filtered_nodes
 
@@ -134,12 +141,14 @@ def print_nodes_with_indent(nodes, indent=0):
 
 
 if __name__ == '__main__':
-    with open('chrome_desktop_example_1.xml', 'r', encoding='utf-8') as f:
-        xml_file_str = f.read()
+    import json
+    with open('2.json', 'r', encoding='utf-8') as f:
+        xml_file_str = json.load(f)["AT"]
     filtered_nodes = filter_nodes(find_leaf_nodes(xml_file_str))
     print(len(filtered_nodes))
-    masks = draw_bounding_boxes(filtered_nodes, 'screenshot.png',
-                                'chrome_desktop_example_1_tagged_remove.png', )
+    masks = draw_bounding_boxes( filtered_nodes, '2.png'
+                               , '2.a.png'
+                               )
 
     # print(masks)
     print(len(masks))

From 59348e31343a969a3a6786bc362ee85fd23e0a3c Mon Sep 17 00:00:00 2001
From: tsuky_chen <3107760494@qq.com>
Date: Sun, 10 Mar 2024 23:49:28 +0800
Subject: [PATCH 2/2] fix multi apps

---
 .../multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json      | 2 +-
 .../multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json b/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json
index 9a2f5ae..6841447 100644
--- a/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json
+++ b/evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json
@@ -45,7 +45,7 @@
       "rules": {
         "include": [],
         "exclude": [
-          "ModuleNotFoundError: No module named"
+          "Error:"
         ]
       }
     }
diff --git a/evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json b/evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json
index cba0a02..5f1750f 100644
--- a/evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json
+++ b/evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json
@@ -1,7 +1,7 @@
 {
   "id": "b337d106-053f-4d37-8da0-7f9c4043a66b",
   "snapshot": "os",
-  "instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the internet for a tutorial on adding line numbers in Vim and setting it as default for my local Vim.",
+  "instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the Internet for a tutorial on adding absolute line numbers in Vim and setting it as default for my local Vim.",
   "source": "authors",
   "config": [
     {
@@ -52,7 +52,7 @@
         "parameters": {
           "files": [
             {
-              "url": "https://drive.usercontent.google.com/download?id=1CyhWjUS2oov4Fzc0VRwTh6LiS2Qu-T_8&export=download&authuser=0&confirm=t&uuid=9d0e2c62-895c-4bb3-a057-30cae60329ed&at=APZUnTVngSwARjYsWSmhSyHAqwID:1709647023362",
+              "url": "https://drive.usercontent.google.com/download?id=1CyhWjUS2oov4Fzc0VRwTh6LiS2Qu-T_8&export=download&authuser=0&confirm=t&uuid=384ea31d-c9ae-4e81-be19-42035c563014&at=APZUnTU9lsYwMLfWb7RIizGr1D7H:1710085473758",
               "path": "eval.sh"
             }
           ]