Merge branch 'main' of github.com:xlang-ai/OSWorld

desktop_env/evaluators/metrics/general.py

@@ -178,18 +178,43 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float:
     return float(all(expect_metrics) and unexpect_metric)
 
 
-_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
+_accessibility_ns_map = {
-    , "attr": "uri:deskat:attributes.at-spi.gnome.org"
+    "ubuntu": {
-    , "cp": "uri:deskat:component.at-spi.gnome.org"
+        "st": "https://accessibility.ubuntu.example.org/ns/state",
-    , "doc": "uri:deskat:document.at-spi.gnome.org"
+        "attr": "https://accessibility.ubuntu.example.org/ns/attributes",
-    , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
+        "cp": "https://accessibility.ubuntu.example.org/ns/component",
-    , "txt": "uri:deskat:text.at-spi.gnome.org"
+        "doc": "https://accessibility.ubuntu.example.org/ns/document",
-    , "val": "uri:deskat:value.at-spi.gnome.org"
+        "docattr": "https://accessibility.ubuntu.example.org/ns/document/attributes",
-    , "act": "uri:deskat:action.at-spi.gnome.org"
+        "txt": "https://accessibility.ubuntu.example.org/ns/text",
-                         }
+        "val": "https://accessibility.ubuntu.example.org/ns/value",
+        "act": "https://accessibility.ubuntu.example.org/ns/action",
+    },
+    "windows": {
+        "st": "https://accessibility.windows.example.org/ns/state",
+        "attr": "https://accessibility.windows.example.org/ns/attributes",
+        "cp": "https://accessibility.windows.example.org/ns/component",
+        "doc": "https://accessibility.windows.example.org/ns/document",
+        "docattr": "https://accessibility.windows.example.org/ns/document/attributes",
+        "txt": "https://accessibility.windows.example.org/ns/text",
+        "val": "https://accessibility.windows.example.org/ns/value",
+        "act": "https://accessibility.windows.example.org/ns/action",
+        "class": "https://accessibility.windows.example.org/ns/class"
+    },
+    "macos": {
+        "st": "https://accessibility.macos.example.org/ns/state",
+        "attr": "https://accessibility.macos.example.org/ns/attributes",
+        "cp": "https://accessibility.macos.example.org/ns/component",
+        "doc": "https://accessibility.macos.example.org/ns/document",
+        "txt": "https://accessibility.macos.example.org/ns/text",
+        "val": "https://accessibility.macos.example.org/ns/value",
+        "act": "https://accessibility.macos.example.org/ns/action",
+        "role": "https://accessibility.macos.example.org/ns/role",
+    }
+
+}
 
 
-def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
+def check_accessibility_tree(result: str, rules: List[Dict[str, Any]], osname: str = "ubuntu") -> float:
     """
     Args:
         result (str): XML of GNOME Accessibility Tree
@@ -205,18 +230,21 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
             "exact": bool specifying whether exact match or fuzzy match should
               be performed. defaults to True.
           }
+        osname (str): "ubuntu" | "windows" | "macos". "ubuntu" by default.
 
     Returns:
         float
     """
 
+    a11y_ns_map = _accessibility_ns_map[osname]
+
     at: _Element = lxml.etree.fromstring(result)
     total_match_score = 1.
     for r in rules:
         if "xpath" in r:
-            elements: List[_Element] = at.xpath(r["xpath"], namespaces=_accessibility_ns_map)
+            elements: List[_Element] = at.xpath(r["xpath"], namespaces=a11y_ns_map)
         elif "selectors" in r:
-            selector = CSSSelector(", ".join(r["selectors"]), namespaces=_accessibility_ns_map)
+            selector = CSSSelector(", ".join(r["selectors"]), namespaces=a11y_ns_map)
             elements: List[_Element] = selector(at)
         else:
             raise ValueError("At least one of xpath and selectors is required")
@@ -403,7 +431,7 @@ def compare_time_in_speedtest_results(speedtest_result_path, time_diff):
         return 0
 
     # open the speedtest results file(csv)
-    date_col = None
+    #date_col = None
     try:
         with open(speedtest_result_path, 'r') as f:
             for i, line in enumerate(f):

evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json

@@ -9,6 +9,7 @@
       "parameters": {
         "command": [
           "google-chrome",
+          "--proxy-server=http://127.0.0.1:18888",
           "--remote-debugging-port=1337"
         ]
       }

evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json

@@ -1,7 +1,7 @@
 {
   "id": "8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f",
   "snapshot": "base_setup",
-  "instruction": "Can you make the video fill up the whole screen? It's a bit too small to see right now, and I'd like to see it better.",
+  "instruction": "Can you enable fullscreen mode in VLC so that the video fill up the whole screen?",
   "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
   "config": [
     {

evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json

@@ -1,7 +1,7 @@
 {
   "id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
   "snapshot": "base_setup",
-  "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
+  "instruction": "Could you convert the song from this music video as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
   "source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
   "config": [
     {