ver Jan11th

finally set up a simple task, or which should be simple

desktop_env/controllers/setup.py

@@ -6,12 +6,13 @@ import uuid
 import os.path
 
 from typing import Dict, List
-from typing import Any, Union
+from typing import Any, Union, Optional
 
 import logging
 logger = logging.getLogger("desktopenv.setup")
 
 import traceback
+import time
 
 class SetupController:
     def __init__(self, http_server: str, cache_dir: str):
@@ -45,6 +46,8 @@ class SetupController:
             assert hasattr(self, setup_function)
             getattr(self, setup_function)(**parameters)
 
+            logger.info("SETUP: %s(%s)", setup_function, str(parameters))
+
         # self._download_setup(config)
         # self._change_wallpaper(config)
         # self._tidy_desktop(config) todo: implement this
@@ -219,35 +222,60 @@ class SetupController:
         except requests.exceptions.RequestException as e:
             logger.error("An error occurred while trying to send the request: %s", e)
 
-    def _execute_setup(self, command: List[str], stdout: str = "", stderr: str = ""):
+    def _execute_setup( self, command: List[str]
+                      , stdout: str = "", stderr: str = ""
+                      , shell: bool = False, until: Optional[Dict[str, Any]] = None):
         if not command:
             raise Exception("Empty comman to launch.")
 
-        payload = json.dumps({"command": command})
+        until: Dict[str, Any] = until or {}
+        terminates: bool = False
+        nb_failings = 0
+
+        payload = json.dumps({"command": command, "shell": shell})
         headers = {"Content-Type": "application/json"}
 
-        try:
+        while not terminates:
-            response = requests.post(self.http_server_setup_root + "/execute", headers=headers, data=payload)
+            try:
-            if response.status_code == 200:
+                response = requests.post(self.http_server_setup_root + "/execute", headers=headers, data=payload)
-                results: Dict[str, str] = response.json()
+                if response.status_code == 200:
-                if stdout:
+                    results: Dict[str, str] = response.json()
-                    with open(os.path.join(self.cache_dir, stdout), "w") as f:
+                    if stdout:
-                        f.write(results["output"])
+                        with open(os.path.join(self.cache_dir, stdout), "w") as f:
-                if stderr:
+                            f.write(results["output"])
-                    with open(os.path.join(self.cache_dir, stderr), "w") as f:
+                    if stderr:
-                        f.write(results["error"])
+                        with open(os.path.join(self.cache_dir, stderr), "w") as f:
-                logger.info( "Command executed successfully: %s -> %s"
+                            f.write(results["error"])
-                           , " ".join(command)
+                    logger.info( "Command executed successfully: %s -> %s"
-                           , response.text
+                               , " ".join(command)
-                           )
+                               , response.text
-            else:
+                               )
-                logger.error("Failed to launch application. Status code: %s", response.text)
+                else:
-        except requests.exceptions.RequestException as e:
+                    logger.error("Failed to launch application. Status code: %s", response.text)
-            logger.error("An error occurred while trying to send the request: %s", e)
+                    results = None
-            traceback.print_exc()
+                    nb_failings += 1
+            except requests.exceptions.RequestException as e:
+                logger.error("An error occurred while trying to send the request: %s", e)
+                traceback.print_exc()
 
-    def _command_setup(self, command: List[str], stdout: str = "", stderr: str = ""):
+                results = None
-        self._execute_setup(command, stdout, stderr)
+                nb_failings += 1
+
+            if len(until)==0:
+                terminates = True
+            elif results is not None:
+                terminates = "returncode" in until and results["returncode"]==until["returncode"]\
+                          or "stdout" in until and until["stdout"] in results["output"]\
+                          or "stderr" in until and until["stderr"] in results["error"]
+            terminates = terminates or nb_failings>=5
+            if not terminates:
+                time.sleep(0.3)
+
+    def _command_setup(self, command: List[str], **kwargs):
+        self._execute_setup(command, **kwargs)
+
+    def _sleep_setup(self, seconds: float):
+        time.sleep(seconds)
 
     def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]):
         # TODO

desktop_env/evaluators/metrics/__init__.py

@@ -7,3 +7,4 @@ from .pdf import check_pdf_pages
 from .libreoffice import check_libre_locale
 from .vlc import is_vlc_playing, is_vlc_recordings_folder
 from .general import check_csv, check_accessibility_tree, check_list
+from .thunderbird import check_thunderbird_prefs

desktop_env/evaluators/metrics/general.py

@@ -13,7 +13,7 @@ from rapidfuzz import fuzz
 import functools
 import re
 
-def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
+def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool:
     return all(k in item and item[k]==val for k, val in pattern.items())
 
 def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:

desktop_env/evaluators/metrics/thunderbird.py

@@ -1,43 +1,91 @@
-#from playwright.sync_api import sync_playwright, Browser
+from typing import List, Pattern, Dict, Match
-#from marionette_driver.marionette import Marionette
+from typing import Union, Any, Optional, Iterable, TypeVar
-#import marionette
-#import pyatspi
 
-import lxml.etree
+import re
-from lxml.cssselect import CSSSelector
+import functools
-from lxml.etree import _Element
+import operator
+import json
 
-from typing import List
+import logging
+logger = logging.getLogger("desktopenv.metric.thunderbird")
+
+V = TypeVar("Value")
+
+def _match_pref(value: Any, rule: Dict[str, Union[str, Any]]) -> bool:
+    #  function _match_pref {{{ # 
+    if rule["method"].startswith("re"):
+        flags: List[str] = rule["method"].split(".")[1:]
+        flags: Iterable[re.RegexFlag] = (getattr(re, fl) for fl in flags)
+        flag: re.RegexFlag = functools.reduce(operator.or_, flags, re.RegexFlag(0))
+        logger.debug("REFLAG: %s", repr(flag))
+
+        match_: Optional[Match[str]] = re.search(rule["ref"], value, flag)
+        return match_ is not None
+    if rule["method"] in { "eq", "ne"
+                         , "le", "lt"
+                         , "ge", "gt"
+                         }:
+        return getattr(operator, rule["method"])(value, rule["ref"])
+    raise NotImplementedError()
+    #  }}} function _match_pref # 
+
+_pref_pattern: Pattern[str] = re.compile(r'^user_pref\("(?P<key>(?:[^"]|\\")+)\", (?P<val>.+)\);$');
+def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any]]]):
+    """
+    Args:
+        result (str): path to result file
+        rule (Dict[str, Dict[str, Dict[str, Any]]]): dict like
+          {
+            "expect": {
+                str: {
+                    "method": str
+                    "ref": something
+                }
+            }
+            "unexpect": {
+                str: {
+                    "method": str
+                    "ref": something
+                }
+            }
+          }
+
+    Returns:
+        float
+    """
+
+    if result is None:
+        return 0.
+
+    expect_rules = rule.get("expect", {})
+    unexpect_rules = rule.get("unexpect", {})
+
+    expect_metrics = {k: False for k in expect_rules}
+    unexpect_metric = True
+    with open(result) as f:
+        for l in f:
+            match_: Match[str] = _pref_pattern.match(l.strip())
+            if match_ is None:
+                continue
+
+            key: str = match_.group("key")
+            #value: str = match_.group("val")
+            #if value in {"true", "false"}:
+                #value = value.title()
+            #value: V = eval(value)
+            value = json.loads(match_.group("val"))
+            if key in expect_rules:
+                logger.debug("K: %s, V: %s", key, repr(value))
+                expect_metrics[key] = _match_pref(value, expect_rules[key])
+            elif key in unexpect_rules:
+                unexpect_metric = unexpect_metric and not _match_pref(value, unexpect_rules[key])
+
+    return float(all(expect_metrics.values()) and unexpect_metric)
 
 if __name__ == "__main__":
-    #with sync_playwright() as plwr:
+    import lxml.etree
-        #while True:
+    from lxml.cssselect import CSSSelector
-            ##try:
+    from lxml.etree import _Element
-            #thunderbird: Browser = plwr.firefox.connect("http://127.0.0.1:6000", timeout=60)
-            #break
-            ##except:
-                ##pass
-        #for ctx in thunderbird.contexts:
-            #for p in ctx.pages:
-                #print(p.url)
-
-    #thunderbird = Marionette()
-    #print(thunderbird.start_session())
-    #print(thunderbird.chrome_window_handles)
-    #print(thunderbird.window_handles)
-    #print(thunderbird.current_chrome_window_handle)
-    #thunderbird.set_context(Marionette.CONTEXT_CONTENT)
-    #print(thunderbird.current_window_handle)
-    #thunderbird.switch_to_window(thunderbird.chrome_window_handles[0])
-    #thunderbird.switch_to_default_content()
-    #thunderbird.switch_to_frame()
-    #print(thunderbird.get_url())
-    #print(thunderbird.get_window_type())
-    #thunderbird.fullscreen()
-    #print(thunderbird.close())
-
-    #registry = pyatspi.Registry.get_default()
-    #registry
 
     #xml = "../../任务数据/Thunderbird/vertical-card-view.xml"
     xml = "../../任务数据/Thunderbird/vertical-table-view.xml"

desktop_env/server/main.py

@@ -48,7 +48,8 @@ def execute_command():
         return jsonify({
             'status': 'success',
             'output': result.stdout,
-            'error': result.stderr
+            'error': result.stderr,
+            'returncode': result.returncode
         })
     except Exception as e:
         return jsonify({

evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json

@@ -1,7 +1,7 @@
 {
   "id": "06fe7178-4491-4589-810f-2e2bc9502122",
   "snapshot": "thunderbird",
-  "instruction": "Could you help me back up all the email files in my profile to ~/email.bak? Please save them in eml format.",
+  "instruction": "Could you help me back up all the email files in my profile to ~/email.bak? Please save them separately in eml format.",
   "source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird",
   "config": [
     {

evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json

@@ -0,0 +1,75 @@
+{
+  "id": "6766f2b8-8a72-417f-a9e5-56fcaa735837",
+  "snapshot": "thunderbird",
+  "instruction": "Set up a signature using my name and affiliation. My name is Anonym and my affiliation is XYZ Lab.",
+  "source": "https://www.adsigner.com/user-manual/signatures/setup-email-client-thunderbird/#:~:text=is%20probably%20hidden.-,Right%20click%20on%20the%20empty%20space%20at%20the%20top%20of,signature%20from%20a%20file%20instead.",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+            "path": "/home/user/thunderbird-profile.tar.gz"
+          }
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "tar",
+          "-xzv",
+          "--recursive-unlink",
+          "-f",
+          "/home/user/thunderbird-profile.tar.gz",
+          "-C",
+          "/home/user/"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "/usr/bin/thunderbird"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/6766f2b8-8a72-417f-a9e5-56fcaa735837",
+  "related_apps": [
+    "thunderbird"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "command",
+        "parameters": {
+          "command": ["wmctrl", "-xFc", "Mail.thunderbird"],
+          "until": {
+            "returncode": 1
+          }
+        }
+      }
+    ],
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
+      "dest": "thunder-prefs.js"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "expect": {
+          "mail.identity.id1.htmlSigText": {
+            "method": "re.S",
+            "ref": "Anonym.+XYZ Lab"
+          }
+        }
+      }
+    },
+    "func": "check_thunderbird_prefs"
+  }
+}

evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup

@@ -0,0 +1,53 @@
+{
+  "id": "6766f2b8-8a72-417f-a9e5-56fcaa735837",
+  "snapshot": "thunderbird",
+  "instruction": "Set up a signature using my name and affiliation. My name is Anonym and my affiliation is XYZ Lab.",
+  "source": "https://www.adsigner.com/user-manual/signatures/setup-email-client-thunderbird/#:~:text=is%20probably%20hidden.-,Right%20click%20on%20the%20empty%20space%20at%20the%20top%20of,signature%20from%20a%20file%20instead.",
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+            "path": "/home/user/thunderbird-profile.tar.gz"
+          }
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/6766f2b8-8a72-417f-a9e5-56fcaa735837",
+  "related_apps": [
+    "thunderbird"
+  ],
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "command",
+        "parameters": {
+          "command": ["wmctrl", "-xFc", "Mail.thunderbird"],
+          "until": {
+            "returncode": 1
+          }
+        }
+      }
+    ],
+    "result": {
+      "type": "vm_file",
+      "path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
+      "dest": "thunder-prefs.js"
+    },
+    "expected": {
+      "type": "rule",
+      "rules": {
+        "expect": {
+          "mail.identity.id1.htmlSigText": {
+            "method": "re.S",
+            "ref": "Anonym.+XYZ Lab"
+          }
+        }
+      }
+    },
+    "func": "check_thunderbird_prefs"
+  }
+}

main.py

@@ -44,9 +44,9 @@ def human_agent():
     Runs the Gym environment with human input.
     """
 
-    with open("evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json", "r") as f:
+    with open("evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup", "r") as f:
         example = json.load(f)
-    example["snapshot"] = "Snapshot 11"
+    example["snapshot"] = "Snapshot 17"
 
     env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
                     , action_space="computer_13"
@@ -92,7 +92,7 @@ def human_agent():
 
     #input("PAUSING")
 
-    env.close()
+    #env.close()
     logger.info("Environment closed.")