Merge branch 'zdy'

desktop_env/evaluators/getters/misc.py

@@ -1,5 +1,5 @@
 import logging
-from typing import TypeVar
+from typing import TypeVar, Dict
 from datetime import datetime, timedelta
 
 logger = logging.getLogger("desktopenv.getters.misc")
@@ -74,13 +74,13 @@ relativeTime_to_IntDay = {
     "first monday four months later": "special"
 }
 
-def get_rule(env, config: R) -> R:
+def get_rule(env, config: Dict[str, R]) -> R:
     """
     Returns the rule as-is.
     """
     return config["rules"]
 
-def get_rule_relativeTime(env, config: R) -> R:
+def get_rule_relativeTime(env, config: Dict[str, R]) -> R:
     """
     According to the rule definded in funciton "apply_rules_to_timeFormat", convert the relative time to absolute time.
     config:

desktop_env/evaluators/metrics/general.py

@@ -1,6 +1,7 @@
 import csv
 import functools
 import json
+import yaml
 import operator
 import re
 import sqlite3
@@ -132,11 +133,11 @@ _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
                          }
 
 
-def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
+def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
     """
     Args:
         result (str): XML of GNOME Accessibility Tree
-        rules (Dict[str, Any]): dict like
+        rules (List[Dict[str, Any]]): list of dict like
           {
             "selectors": list of str as CSS selectors, will be connected by ", "
               to form a composite selector. Only one from `selectors` and
@@ -154,30 +155,33 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
     """
 
     at: _Element = lxml.etree.fromstring(result)
-    if "xpath" in rules:
+    total_match_score = 1.
-        elements: List[_Element] = at.xpath(rules["xpath"], namespaces=_accessibility_ns_map)
+    for r in rules:
-    elif "selectors" in rules:
+        if "xpath" in r:
-        selector = CSSSelector(", ".join(rules["selectors"]), namespaces=_accessibility_ns_map)
+            elements: List[_Element] = at.xpath(r["xpath"], namespaces=_accessibility_ns_map)
-        elements: List[_Element] = selector(at)
+        elif "selectors" in r:
-    else:
+            selector = CSSSelector(", ".join(r["selectors"]), namespaces=_accessibility_ns_map)
-        raise ValueError("At least one of xpath and selectors is required")
+            elements: List[_Element] = selector(at)
+        else:
+            raise ValueError("At least one of xpath and selectors is required")
 
-    if len(elements) == 0:
+        if len(elements) == 0:
-        print("no elements")
+            print("no elements")
-        return 0.
+            return 0.
 
-    if "text" in rules:
+        if "text" in r:
-        match_func: Callable[[str], Number] = functools.partial(operator.eq if rules["exact"] \
+            match_func: Callable[[str], Number] = functools.partial( operator.eq if r["exact"] \
-                                                                    else (lambda a, b: fuzz.ratio(a, b) / 100.)
+                                                                               else (lambda a, b: fuzz.ratio(a, b) / 100.)
-                                                                , rules["text"]
+                                                                   , r["text"]
-                                                                )
+                                                                   )
-        match_score: Number = 0
+            match_score: Number = 0
-        for elm in elements:
+            for elm in elements:
-            match_score = max(match_score, match_func(elm.text or None))
+                match_score = max(match_score, match_func(elm.text or None))
-    else:
+        else:
-        match_score = 1.
+            match_score = 1.
+        total_match_score *= match_score
 
-    return float(match_score)
+    return float(total_match_score)
 
 
 # def check_existence(result: str, *args) -> float:
@@ -189,7 +193,7 @@ def run_sqlite3(result: str, rules: Dict[str, Any]) -> float:
     return float(cursor.fetchone()[0] or 0)
 
 
-def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str]]]]) -> float:
+def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str]]]], is_yaml: bool = False) -> float:
     """
     Args:
         result (str): path to json file
@@ -204,6 +208,7 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
             ],
             "unexpect": <the same as `expect`
           }
+        is_yaml (bool): yaml rather than json
 
     Returns:
         float
@@ -212,7 +217,10 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
     if result is None:
         return 0.
     with open(result) as f:
-        result: Dict[str, Any] = json.load(f)
+        if is_yaml:
+            result: Dict[str, Any] = yaml.load(f, Loader=yaml.Loader)
+        else:
+            result: Dict[str, Any] = json.load(f)
 
     expect_rules = rules.get("expect", {})
     unexpect_rules = rules.get("unexpect", {})

desktop_env/evaluators/metrics/table.py

@@ -12,6 +12,7 @@ import pandas as pd
 from openpyxl import Workbook
 from openpyxl.cell.cell import Cell
 from openpyxl.worksheet.cell_range import MultiCellRange
+from openpyxl.utils import get_column_letter
 from openpyxl.worksheet.datavalidation import DataValidation
 from openpyxl.worksheet.worksheet import Worksheet
 
@@ -208,8 +209,10 @@ def compare_table(result: str, expected: str = None, **options) -> float:
             for rl in r["rules"]:
                 for rng in MultiCellRange(rl["range"]):
                     for cdn in rng.cells:
-                        value1: str = str(read_cell_value(*sheet1, cdn))
+                        coordinate: str = "{:}{:d}".format(get_column_letter(cdn[1]), cdn[0])
-                        value2: str = str(read_cell_value(*sheet2, cdn))
+                        value1: str = str(read_cell_value(*sheet1, coordinate))
+                        value2: str = str(read_cell_value(*sheet2, coordinate))
+                        logger.debug("%s: %s vs %s", cdn, value1, value2)
 
                         for rplc in rl.get("normalization", []):
                             value1 = value1.replace(rplc[0], rplc[1])
@@ -230,11 +233,11 @@ def compare_table(result: str, expected: str = None, **options) -> float:
 
                         if rl["type"]=="includes":
                             metric: bool = value1 in value2
-                        if rl["type"]=="includes_by":
+                        elif rl["type"]=="includes_by":
                             metric: bool = value2 in value1
-                        if rl["type"]=="fuzzy_match":
+                        elif rl["type"]=="fuzzy_match":
                             metric: bool = fuzz.ratio(value1, value2) >= rl.get("threshold", 85.)
-                        if rl["type"]=="exact_match":
+                        elif rl["type"]=="exact_match":
                             metric: bool = value1==value2
                         total_metric = total_metric and metric
 

evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json

@@ -0,0 +1,147 @@
+{
+  "id": "415ef462-bed3-493a-ac36-ca8c6d23bf1b",
+  "snapshot": "thunderbird",
+  "instruction": "Save the AWS invoice of December from the email. I have moved that email to local \"Bills\" folder. Save it to the my receipts folder. Keep the file name pattern and update a record to my tally book.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["mkdir", "-p", "/home/user/Documents/Finance/receipts", "/home/user/Documents/Projects"]
+      }
+    },
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2308.pdf", "url": "https://drive.google.com/uc?id=1azRFXf4A7fvW0S7r9upHvleMEi-92hHM&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2309.pdf", "url": "https://drive.google.com/uc?id=1x-lpHm8U4U7uRPZ74-9wq9KzW2R55ln1&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2310.pdf", "url": "https://drive.google.com/uc?id=1pcrgV9G6NO4ekMEQBiupwXtq6mmke7b_&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/aws-invoice-2311.pdf", "url": "https://drive.google.com/uc?id=1JzbCK_nIY8X_3QZjnkzTtb-cRoq9zNT-&export=download"},
+          {"path": "/home/user/Documents/Finance/receipts/X-receipt-2312.pdf", "url": "https://drive.google.com/uc?id=1QzWjNzvNosG_yQr7VVonvYb3cUYF5f3u&export=download"},
+          {"path": "/home/user/Documents/Finance/tally_book.xlsx", "url": "https://drive.google.com/uc?id=13yuLhBPmouoWR-DybfgaIbWUOxbY_jhL&export=download"},
+          {"path": "/home/user/.projects.tar.xz", "url": "https://drive.google.com/uc?id=1oJcxpjqF474Wm16i1aZc8DlCEfAvc4t_&export=download"},
+					{
+						"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+						"path": "/home/user/thunderbird-profile.tar.gz"
+					}
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["tar", "-xJvf", "/home/user/.projects.tar.xz", "-C", "/home/user/Documents/Projects"]
+      }
+    },
+		{
+			"type": "execute",
+			"parameters": {
+				"command": [
+					"tar",
+					"-xzv",
+					"--recursive-unlink",
+					"-f",
+					"/home/user/thunderbird-profile.tar.gz",
+					"-C",
+					"/home/user/"
+				]
+			}
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": ["thunderbird"]
+      }
+    }
+  ],
+  "trajectory": "trajectories/415ef462-bed3-493a-ac36-ca8c6d23bf1b",
+  "related_apps": ["thunderbird", "libreoffice_calc", "os"],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "activate_window",
+        "parameters": {
+          "window_name": "tally_book.xlsx - LibreOffice Calc",
+          "strict": true
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python",
+            "-c",
+            "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
+          ]
+        }
+      },
+      {
+        "type": "sleep",
+        "parameters": {
+          "seconds": 0.5
+        }
+      },
+      {
+        "type": "download",
+        "parameters": {
+          "files": [
+            {"path": "/home/user/.aws-invoice-2312.pdf", "url": "https://drive.google.com/uc?id=1RqbulzKG_HeYb1GZmLABOzlohlFg02UU&export=download"}
+          ]
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": ["diff", ".aws-invoice-2312.pdf", "/home/user/Documents/Finance/receipts/aws-invoice-2312.pdf"],
+          "stdout": "diff.out"
+        }
+      }
+    ],
+    "func": ["compare_table", "check_list"],
+    "result": [
+      {
+        "type": "vm_file",
+        "path": "/home/user/Documents/Finance/tally_book.xlsx",
+        "dest": "tally_book.xlsx"
+      },
+      {
+        "type": "cache_file",
+        "path": "diff.out"
+      }
+    ],
+    "expected": [
+      {
+        "type": "cloud_file",
+        "path": "https://drive.google.com/uc?id=1x8m-korGI1PhJm8PAQVTlWYKneK4WKvn&export=download",
+        "dest": "tally_book_gt.xlsx"
+      },
+      {
+        "type": "rule",
+        "rules": {
+          "unexpect": [
+            ".+"
+          ]
+        }
+      }
+    ],
+    "options": [
+      {
+        "rules": [
+          {
+            "type": "sheet_data",
+            "sheet_idx0": 0,
+            "sheet_idx1": "EI0"
+          }
+        ]
+      },
+      {}
+    ]
+  }
+}

evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json

@@ -1,7 +1,7 @@
 {
 	"id": "b5062e3e-641c-4e3a-907b-ac864d2e7652",
 	"snapshot": "libreoffice_calc",
-	"instruction": "I've got a mass of different categories of paper in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".",
+	"instruction": "I've got a mass of papers in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".",
 	"source": "authors",
 	"config": [
 		{

evaluation_examples/examples/multi_apps/e1fc0df3-c8b9-4ee7-864c-d0b590d3aa56.json

@@ -0,0 +1,59 @@
+{
+  "id": "e1fc0df3-c8b9-4ee7-864c-d0b590d3aa56",
+  "snapshot": "libreoffice_writer",
+  "instruction": "Install LanguageTool extension for my LibreOffice",
+  "source": "authors",
+  "config": [
+    {
+      "type": "launch",
+      "parameters": {
+        "command": ["libreoffice", "--writer"]
+      }
+    }
+  ],
+  "trajectory": "trajectories/e1fc0df3-c8b9-4ee7-864c-d0b590d3aa56",
+  "related_apps": ["chrome", "libreoffice", "os"],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "command",
+        "parameters": {
+          "command": ["grep", "-nHr", "languagetool", "/home/user/.config/libreoffice/4/user/uno_packages/cache/uno_packages/"],
+          "stdout": "grep.out"
+        }
+      },
+      {
+        "type": "command",
+        "parameters": {
+          "command": ["apt", "list", "--installed"],
+          "stdout": "apt.out"
+        }
+      }
+    ],
+    "func": ["check_list", "check_list"],
+    "result": [
+      {
+        "type": "cache_file",
+        "path": "grep.out"
+      },
+      {
+        "type": "cache_file",
+        "path": "apt.out"
+      }
+    ],
+    "expected": [
+      {
+        "type": "rule",
+        "rules": {
+          "expect": ["org\\.openoffice\\.languagetool\\.oxt"]
+        }
+      },
+      {
+        "type": "rule",
+        "rules": {
+          "expect": ["openjdk-\\d+-(jre|jdk)"]
+        }
+      }
+    ]
+  }
+}

evaluation_examples/examples/multi_apps/e2392362-125e-4f76-a2ee-524b183a3412.json

@@ -0,0 +1,86 @@
+{
+  "id": "e2392362-125e-4f76-a2ee-524b183a3412",
+  "snapshot": "chrome",
+  "instruction": "I recently started using the famous personal academic homepage template from academicpages.github.io to build my own personal homepage, and I have cloned it to my local ~/Code/Website folder. According to an online tutorial, I can configure my name and contact information in the _config.yaml file. However, I am not familiar with the YAML file format. Please help me find the sections related to the name and contact information in this file and change them to “Test Account” and “Test@gmail.com”.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "command",
+      "parameters": {
+        "command": ["mkdir", "-p", "/home/user/Code/Website"]
+      }
+    },
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "path": "/home/user/.tmp.tar.xz",
+            "url": "https://drive.google.com/uc?id=1ordb5kRSPDKgRi7nYQchn8hGt-INELML&export=download"
+          }
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["tar", "-xJvf", ".tmp.tar.xz", "-C", "/home/user/Code/Website/"]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": ["google-chrome", "--remote-debugging-port=1337"]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "socat",
+          "tcp-listen:9222,fork",
+          "tcp:localhost:1337"
+        ]
+      }
+    },
+    {
+      "type": "chrome_open_tabs",
+      "parameters": {
+        "urls_to_open": ["https://academicpages.github.io/"]
+      }
+    }
+  ],
+  "trajectory": "trajectories/e2392362-125e-4f76-a2ee-524b183a3412",
+  "related_apps": ["chrome", "os", "vscode"],
+  "evaluator": {
+    "func": "check_json",
+    "options": {"is_yaml": true},
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "expect": [
+          {
+            "key": ["name"],
+            "method": "eq",
+            "ref": "Test Account"
+          },
+          {
+            "key": ["author", "name"],
+            "method": "eq",
+            "ref": "Test Account"
+          },
+          {
+            "key": ["author", "email"],
+            "method": "eq",
+            "ref": "Test@gmail.com"
+          }
+        ]
+      }
+    },
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/Code/Website/academicpages.github.io/_config.yml",
+      "dest": "_config.yaml"
+    }
+  }
+}

evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json

@@ -0,0 +1,71 @@
+{
+  "id": "f5c13cdd-205c-4719-a562-348ae5cd1d91",
+  "snapshot": "thunderbird",
+  "instruction": "Here is a table recording tuition payments in ~/Documents/Departments/finance. I have already composed an e-mail to remind people who haven't finished payment yet. Please help me to add their emails to the receiver field.",
+  "source": "authors",
+  "config": [
+    {
+      "type": "execute",
+      "parameters": {
+        "command": ["mkdir", "-p", "/home/user/Documents/Departments/finance"]
+      }
+    },
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+            "path": "/home/user/thunderbird-profile.tar.gz"
+          },
+          {
+            "url": "https://drive.google.com/uc?id=1k0_69RKAx18bEX00EJXtWmSwdfNVP9NA&export=download",
+            "path": "/home/user/.payment-reminder-mail-body.html"
+          },
+          {
+            "url": "https://drive.google.com/uc?id=1nNc0NoOuP3Of0eGsKY-1kctg63vIjXl5&export=download",
+            "path": "/home/user/Documents/Departments/finance/tuition.xlsx"
+          }
+        ]
+      }
+    },
+		{
+			"type": "execute",
+			"parameters": {
+				"command": [
+					"tar",
+					"-xzv",
+					"--recursive-unlink",
+					"-f",
+					"/home/user/thunderbird-profile.tar.gz",
+					"-C",
+					"/home/user/"
+				]
+			}
+		},
+		{
+			"type": "launch",
+			"parameters": {
+				"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
+				"shell": true
+			}
+		}
+  ],
+  "trajectory": "trajectories/f5c13cdd-205c-4719-a562-348ae5cd1d91",
+  "related_apps": ["thunderbird", "os", "libreoffice_calc"],
+  "evaluator": {
+    "func": "check_accessibility_tree",
+    "result": {
+      "type": "accessibility_tree"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": [
+        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"]},
+        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"]},
+        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"]},
+        {"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"]}
+      ]
+    }
+  }
+}

evaluation_examples/examples/multi_apps/f8369178-fafe-40c2-adc4-b9b08a125456.json

@@ -0,0 +1,31 @@
+{
+  "id": "f8369178-fafe-40c2-adc4-b9b08a125456",
+  "snapshot": "chrome",
+  "instruction": "Help me to install Orchis theme from gnome-look.org and change to it for my GNOME desktop.",
+  "source": "https://itsfoss.com/install-switch-themes-gnome-shell",
+  "config": [],
+  "trajectory": "trajectories/f8369178-fafe-40c2-adc4-b9b08a125456",
+  "related_apps": ["chrome", "os"],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "execute",
+        "parameters": {
+          "command": ["gsettings", "get", "org.gnome.desktop.interface", "gtk-theme"],
+          "stdout": "gsettings.out"
+        }
+      }
+    ],
+    "func": "check_list",
+    "result": {
+      "type": "cache_file",
+      "path": "gsettings.out"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "expect": ["Orchis"]
+      }
+    }
+  }
+}

evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json

@@ -48,11 +48,13 @@
     },
     "expected": {
       "type": "rule",
-      "rules": {
+      "rules": [
-        "selectors": [
+        {
-          "application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]"
+          "selectors": [
-        ]
+            "application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]"
-      }
+          ]
+        }
+      ]
     },
     "func": "check_accessibility_tree"
   }

requirements.txt

@@ -42,3 +42,4 @@ func-timeout
 beautifulsoup4
 dashscope
 google-generativeai
+PyYaml