Merge pull request #9 from xlang-ai/xiaochuanli/addChromeExtensions

Xiaochuanli/add chrome extensions

desktop_env/evaluators/getters/chrome.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import platform
 import time
 import sqlite3
 from typing import Dict, Any, List
@@ -393,10 +394,11 @@ def get_active_url_from_accessTree(env, config):
         This function is used to get the active tab url from the accessibility tree.
         config: 
             Dict[str, str]{
-                'xpath': 
+                # we no longer need to specify the xpath or selectors, since we will use defalut value
-                    the same as in metrics.general.accessibility_tree.
+                # 'xpath': 
-                'selectors': 
+                #     the same as in metrics.general.accessibility_tree.
-                    the same as in metrics.general.accessibility_tree.
+                # 'selectors': 
+                #     the same as in metrics.general.accessibility_tree.
                 'goto_prefix':
                     the prefix you want to add to the beginning of the url to be opened, default is "https://",
                     (the url we get from accTree does not have prefix)
@@ -410,14 +412,17 @@ def get_active_url_from_accessTree(env, config):
     logger.debug("AT@eval: %s", accessibility_tree)
     # first, use accessibility API to get the active tab URL
     at: _Element = lxml.etree.fromstring(accessibility_tree)
-    if "xpath" in config:
+    arch = platform.machine()
-        elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map)
+    if "arm" in arch:
-    elif "selectors" in config:
+        selector = CSSSelector(", application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]", namespaces=_accessibility_ns_map)
-        selector = CSSSelector(", ".join(config["selectors"]), namespaces=_accessibility_ns_map)
-        elements: List[_Element] = selector(at)
     else:
-        raise ValueError("At least one of xpath and selectors is required")
+        selector = CSSSelector(", application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]", namespaces=_accessibility_ns_map)
-
+    elements: List[_Element] = selector(at)
+    # if "xpath" in config:
+    #     elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map)
+    # elif "selectors" in config:
+    #     selector = CSSSelector(", ".join(config["selectors"]), namespaces=_accessibility_ns_map)
+    #     elements: List[_Element] = selector(at)
     if len(elements) == 0:
         print("no elements found")
         return 0.

evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json

@@ -46,17 +46,11 @@
         "func":["check_direct_json_object", "check_direct_json_object"],
         "result": [{
             "type": "active_tab_url_parse",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "parse_keys": ["locationName", "dropLocationName", "filterCriteria_carCategory", "filterCriteria_sortBy"]
         },
         {
           "type": "active_tab_html_parse",
-          "selectors":[
-            "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-          ],
           "goto_prefix": "https://www.",
           "category": "xpath",
           "xpathObject":{

evaluation_examples/examples/chrome/2888b4e6-5b47-4b57-8bf5-c73827890774.json

@@ -47,9 +47,6 @@
         "func":"exact_match",
         "result": {
             "type": "url_dashPart",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "partIndex": -1,
             "needDeleteId": true,

evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json

@@ -46,9 +46,6 @@
         "func":["check_direct_json_object", "is_expected_url_pattern_match"],
         "result": [{
             "type": "url_dashPart",
-            "selectors":[
-                "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "partIndex": -2,
             "needDeleteId": false,
@@ -57,9 +54,6 @@
         },
         {
             "type": "active_tab_info",
-            "selectors":[
-                "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www."
         }],
         "expected":[
@@ -82,4 +76,3 @@
         }]
     }
   }
-}

evaluation_examples/examples/chrome/47543840-672a-467d-80df-8f7c3b9788c9.json

@@ -48,16 +48,10 @@
         "result": [
             {
                 "type": "active_url_from_accessTree",
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ],
                 "goto_prefix": "https://www."
             },
             {
                 "type": "active_tab_html_parse",
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ],
                 "goto_prefix": "https://www.",
                 "category": "class",
                 "class_singleObject":{},
@@ -74,9 +68,6 @@
             },
             {
                 "type": "active_tab_html_parse",
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ],
                 "goto_prefix": "https://www.",
                 "category": "xpath",
                 "xpathObject":{

evaluation_examples/examples/chrome/6c4c23a1-42a4-43cc-9db1-2f86ff3738cc.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_html_parse",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "category": "class",
             "class_singleObject":{

evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_url_parse",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "parse_keys": ["q", "tbs"]
         },

evaluation_examples/examples/chrome/82279c77-8fc6-46f6-9622-3ba96f61b477.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_url_parse",
-            "selectors":[
-                "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "parse_keys": ["list_price_max", "maximum_distance", "zip","fuel_slugs[]"]
         },

evaluation_examples/examples/chrome/82bc8d6a-36eb-4d2d-8801-ef714fb1e55a.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_url_parse",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "parse_keys": ["fromStation", "toStation", "departing"],
             "replace":{

evaluation_examples/examples/chrome/9f3f70fc-5afc-4958-a7b7-3bb4fcb01805.json

@@ -47,16 +47,10 @@
         "conj": "and",
         "result": [
             {
-                "type": "active_tab_info",
+                "type": "active_tab_info"
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ]
             },
             {
                 "type": "active_tab_html_parse",
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ],
                 "category": "xpath",
                 "xpathObject":{
                     "/html/body/div[2]/div/div[6]/div[2]/div[2]/div/div[1]/div[4]/ul/li[2]": "money"

evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json

@@ -58,6 +58,22 @@
           "forms\\?title=&field_component_target_id=431"
         ]
       }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+      "chrome"
+    ],
+    "evaluator": {
+        "func":"is_expected_url_pattern_match",
+        "result": {
+            "type": "active_tab_info",
+            "goto_prefix": "https://www."
+        },
+        "expected":{
+            "type": "rule",
+            "rules":{
+                "expected": ["forms\\?title=&field_component_target_id=431"]
+            }
+        }
     }
-  }
 }

evaluation_examples/examples/chrome/a96b564e-dbe9-42c3-9ccf-b4498073438a.json

@@ -56,9 +56,6 @@
         "func":"is_expected_active_tab",
         "result": {
             "type": "active_tab_info",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www."
         },
         "expected":{
@@ -68,73 +65,5 @@
                 "url": "https://discussions.flightaware.com/t/graphs-for-dump1090-my-version-with-install-script/46263"
             }
         }
-=======
-  "id": "a96b564e-dbe9-42c3-9ccf-b4498073438a",
-  "snapshot": "chrome",
-  "instruction": "Find discussions of community and open one with most replies.",
-  "change_possibility": "low",
-  "source": "test_task_0",
-  "config": [
-    {
-      "type": "launch",
-      "parameters": {
-        "command": [
-          "google-chrome",
-          "--remote-debugging-port=1337"
-        ]
-      }
-    },
-    {
-      "type": "launch",
-      "parameters": {
-        "command": [
-          "socat",
-          "tcp-listen:9222,fork",
-          "tcp:localhost:1337"
-        ]
-      }
-    },
-    {
-      "type": "chrome_open_tabs",
-      "parameters": {
-        "urls_to_open": [
-          "https://www.flightaware.com/"
-        ]
-      }
-    },
-    {
-      "type": "activate_window",
-      "parameters": {
-        "window_name": "Google Chrome"
-      }
-    },
-    {
-      "type": "execute",
-      "parameters": {
-        "command": [
-          "python",
-          "-c",
-          "import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
-        ]
       }
     }
-  ],
-  "trajectory": "trajectories/",
-  "related_apps": [
-    "chrome"
-  ],
-  "evaluator": {
-    "func": "is_expected_active_tab",
-    "result": {
-      "type": "active_tab_info"
-    },
-    "expected": {
-      "type": "rule",
-      "rules": {
-        "type": "url",
-        "url": "https://discussions.flightaware.com/t/the-banter-thread/4412"
-      }
-    }
-  }
-}
-  

evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json

@@ -59,6 +59,25 @@
       "rules": {
         "expected": "tamiflu.html#side-effects"
       }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+      "chrome"
+    ],
+    "evaluator": {
+        "func":"exact_match",
+        "result": {
+            "type": "url_dashPart",
+            "goto_prefix": "https://www.",
+            "partIndex": -1,
+            "needDeleteId": false,
+            "returnType": "string"
+        },
+        "expected":{
+            "type": "rule",
+            "rules":{
+                "expected": "tamiflu.html#side-effects"
+            }
+        }
     }
   }
-}

evaluation_examples/examples/chrome/b4f95342-463e-4179-8c3f-193cd7241fb2.json

@@ -47,9 +47,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_html_parse",
-            "selectors":[
-                "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-              ],
             "goto_prefix": "https://www.", 
             "category":"class",
             "class_singleObject":{},

evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_html_parse",
-            "selectors":[
-                "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "category": "xpath",
             "xpathObject":{

evaluation_examples/examples/chrome/c1fa57f3-c3db-4596-8f09-020701085416.json

@@ -56,9 +56,6 @@
         "func":"is_expected_url_pattern_match",
         "result": {
             "type": "active_tab_info",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www."
         },
         "expected":{

evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json

@@ -60,6 +60,22 @@
           "S=4"
         ]
       }
+    ],
+    "trajectory": "trajectories/",
+    "related_apps": [
+      "chrome"
+    ],
+    "evaluator": {
+        "func":"is_expected_url_pattern_match",
+        "result": {
+            "type": "active_url_from_accessTree",
+            "goto_prefix": "https://www."
+        },
+        "expected":{
+            "type": "rule",
+            "rules":{
+                "expected": ["AgeAppropriate:Kids", "search=spider-man%20toys", "S=4"]
+            }
+        }
     }
   } 
-}

evaluation_examples/examples/chrome/da46d875-6b82-4681-9284-653b0c7ae241.json

@@ -47,16 +47,10 @@
         "conj": "and",
         "result": [
             {
-                "type": "active_tab_info",
+                "type": "active_tab_info"
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ]
             },
             {
                 "type": "active_tab_html_parse",
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ],
                 "category": "class",
                 "class_singleObject":{},
                 "class_multiObject":{
@@ -68,9 +62,6 @@
             },
             {
                 "type": "active_tab_html_parse",
-                "selectors": [
-                  "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-                ],
                 "category": "input",
                 "inputObject":{
                     "/html/body/div/div/form/div[7]/div/div/div[1]/input[1]": "name",

evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json

@@ -32,9 +32,6 @@
         "func":"is_expected_url_pattern_match",
         "result": {
             "type": "active_tab_info",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www."
         },
         "expected":{

evaluation_examples/examples/chrome/f79439ad-3ee8-4f99-a518-0eb60e5652b0.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_url_parse",
-            "selectors":[
-                "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-              ],
             "goto_prefix": "https://www.",
             "parse_keys":["originIata", "destinationIata", "tpAdults", "tpTeens", "tpChildren", "tpStartDate", "isReturn"],
             "replace":{

evaluation_examples/examples/chrome/fc6d8143-9452-4171-9459-7f515143419a.json

@@ -46,9 +46,6 @@
         "func":"check_direct_json_object",
         "result": {
             "type": "active_tab_html_parse",
-            "selectors":[
-              "application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
-            ],
             "goto_prefix": "https://www.",
             "category": "class",
             "class_singleObject":{