ver Jan4th

updated interfaces for thunderbird evaluation, not tested

desktop_env/controllers/setup.py

@@ -6,7 +6,7 @@ import uuid
 import os.path
 
 from typing import Dict, List
-from typing import Any
+from typing import Any, Union
 
 
 class SetupController:
@@ -145,7 +145,7 @@ class SetupController:
             print("An error occurred while trying to send the request:", e)
 
     def _tidy_desktop_setup(self, **config):
-        raise NotImplementedError
+        raise NotImplementedError()
 
     def _open_setup(self, path: str):
         # if not config:
@@ -170,3 +170,56 @@ class SetupController:
                 print("Failed to open file. Status code:", response.text)
         except requests.exceptions.RequestException as e:
             print("An error occurred while trying to send the request:", e)
+
+    def _launch_setup(self, command: List[str]):
+        if not command:
+            raise Exception("Empty comman to launch.")
+
+        payload = json.dumps({"command": command})
+        headers = {"Content-Type": "application/json"}
+
+        try:
+            response = requests.post(self.http_server + "/launch", headers=headers, data=payload)
+            if response.status_code == 200:
+                print("Command executed successfully:", response.text)
+            else:
+                print("Failed to launch application. Status code:", response.text)
+        except requests.exceptions.RequestException as e:
+            print("An error occurred while trying to send the request:", e)
+
+    def _execute_setup(self, command: List[str], stdout: str = "", stderr: str = ""):
+        if not command:
+            raise Exception("Empty comman to launch.")
+
+        payload = json.dumps({"command": command})
+        headers = {"Content-Type": "application/json"}
+
+        try:
+            response = requests.post(self.http_server + "/launch", headers=headers, data=payload)
+            if response.status_code == 200:
+                results: Dict[str, str] = response.json()
+                if stdout:
+                    with open(os.path.join(self.cache_dir, stdout), "w") as f:
+                        f.write(results["output"])
+                if stderr:
+                    with open(os.path.join(self.cache_dir, stderr), "w") as f:
+                        f.write(results["error"])
+                print( "Command executed successfully: {:} ->".format(" ".join(command))
+                     , response.text
+                     )
+            else:
+                print("Failed to launch application. Status code:", response.text)
+        except requests.exceptions.RequestException as e:
+            print("An error occurred while trying to send the request:", e)
+
+    def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]):
+        # TODO
+        raise NotImplementedError()
+    def _replay_setup(self, trajectory: str):
+        """
+        Args:
+            trajectory (str): path to the replay trajectory file
+        """
+
+        # TODO
+        raise NotImplementedError()

desktop_env/envs/desktop_env.py

@@ -68,19 +68,8 @@ class DesktopEnv(gym.Env):
         self.cache_dir_base: str = cache_dir
 
         # task-aware stuffs
-        self.snapshot_path = task_config["snapshot"]  # todo: handling the logic of snapshot directory
+        # todo: handling the logic of snapshot directory
-        self.task_id: str = task_config["id"]
+        self._set_task_info(task_config)
-        self.cache_dir: str = os.path.join(self.cache_dir_base, self.task_id)
-        os.makedirs(self.cache_dir, exist_ok=True)
-        self.instruction = task_config["instruction"]
-        self.config = task_config["config"]
-
-        self.evaluator = task_config["evaluator"]
-        self.metric: Metric = getattr(metrics, self.evaluator["func"])
-        self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
-        self.expected_getter: Getter = getattr(getters, "get_{:}".format(
-            self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None
-        self.metric_options: Dict[str, Any] = self.evaluator.get("options", {})
 
         # Initialize emulator and controller
         print("Initializing...")
@@ -151,14 +140,10 @@ class DesktopEnv(gym.Env):
         screenshot_image_path = self._get_screenshot()
         return screenshot_image_path
 
-    def reset(self, task_config: Optional[Dict[str, Any]] = None, seed=None, options=None):
+    def _set_task_info(self, task_config: Dict[str, Any]):
-        print("Resetting environment...")
-
-        print("Switching task...")
-        if task_config is not None:
         self.snapshot_path = task_config["snapshot"]
-            self.task_id = task_config["id"]
+        self.task_id: str = task_config["id"]
-            self.cache_dir = os.path.join(self.cache_dir_base, self.task_id)
+        self.cache_dir: str = os.path.join(self.cache_dir_base, self.task_id)
         os.makedirs(self.cache_dir, exist_ok=True)
         self.instruction = task_config["instruction"]
         self.config = task_config["config"]
@@ -168,8 +153,14 @@ class DesktopEnv(gym.Env):
         self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
         self.expected_getter: Getter = getattr(getters, "get_{:}".format(
             self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None
-            self.metric_options = self.evaluator.get("options", {})
+        self.metric_options: Dict[str, Any] = self.evaluator.get("options", {})
 
+    def reset(self, task_config: Optional[Dict[str, Any]] = None, seed=None, options=None):
+        print("Resetting environment...")
+
+        print("Switching task...")
+        if task_config is not None:
+            self._set_task_info(task_config)
             self.setup_controller.reset_cache_dir(self.cache_dir)
 
         print("Setting counters...")
@@ -228,6 +219,9 @@ class DesktopEnv(gym.Env):
         """
         Evaluate whether the task is successfully completed.
         """
+
+        self.setup_controller.setup(self.evaluator["postconfig"])
+
         result_state = self.result_getter(self, self.evaluator["result"])
         expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \
             else None

desktop_env/evaluators/getters/__init__.py

@@ -1,2 +1,2 @@
-from .file import get_cloud_file, get_vm_file
+from .file import get_cloud_file, get_vm_file, get_cache_file
 from .misc import get_rule

desktop_env/evaluators/getters/file.py

@@ -33,8 +33,6 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
     """
 
     _path = os.path.join(env.cache_dir, config["dest"])
-    if os.path.exists(_path):
-        return _path
 
     file = env.controller.get_file(config["path"])
     with open(_path, "wb") as f:
@@ -42,3 +40,12 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
 
     return _path
 
+def get_cache_file(env, config: Dict[str, str]) -> str:
+    """
+    Config:
+        path (str): relative path in cache dir
+    """
+
+    _path = os.path.join(env.cache_dir, config["path"])
+    assert os.path.exists(_path)
+    return _path

desktop_env/evaluators/getters/misc.py

@@ -1,8 +1,63 @@
 from typing import TypeVar
 
+import platform
+import subprocess
+import ctypes
+import os
+
 R = TypeVar("Rule")
 def get_rule(env, config: R) -> R:
     """
     Returns the rule as-is.
     """
     return config["rules"]
+
+
+def get_desktop_path():
+    username = os.getlogin()  # Get the current username
+    if platform.system() == "Windows":
+        return os.path.join("C:", "Users", username, "Desktop")
+    elif platform.system() == "Darwin":  # macOS is identified as 'Darwin'
+        return os.path.join("/Users", username, "Desktop")
+    elif platform.system() == "Linux":
+        return os.path.join("/home", username, "Desktop")
+    else:
+        raise Exception("Unsupported operating system")
+
+
+def get_wallpaper():
+    def get_wallpaper_windows():
+        SPI_GETDESKWALLPAPER = 0x73
+        MAX_PATH = 260
+        buffer = ctypes.create_unicode_buffer(MAX_PATH)
+        ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0)
+        return buffer.value
+
+    def get_wallpaper_macos():
+        script = """
+        tell application "System Events" to tell every desktop to get picture
+        """
+        process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE)
+        output, error = process.communicate()
+        if error:
+            print("Error:", error)
+        else:
+            return output.strip().decode('utf-8')
+
+    def get_wallpaper_linux():
+        try:
+            output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"])
+            return output.decode('utf-8').strip().replace('file://', '').replace("'", "")
+        except Exception as e:
+            print("Error:", e)
+            return None
+
+    os_name = platform.system()
+    if os_name == 'Windows':
+        return get_wallpaper_windows()
+    elif os_name == 'Darwin':
+        return get_wallpaper_macos()
+    elif os_name == 'Linux':
+        return get_wallpaper_linux()
+    else:
+        return "Unsupported OS"

desktop_env/evaluators/metrics/__init__.py

@@ -5,3 +5,4 @@ from .docs import compare_font_names, compare_subscript_contains, has_page_numbe
 from .docs import is_first_line_centered, check_file_exists, compare_contains_image
 from .pdf import check_pdf_pages
 from .libreoffice import check_libre_locale
+from .general import check_csv

desktop_env/evaluators/metrics/general.py

@@ -1,57 +1,30 @@
-import platform
+import csv
-import subprocess
+from typing import Dict, List
-import ctypes
-import os
 
+def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
+    return all(k in item and item[k]==val for k, val in pattern.items())
 
-# todo: move to getter module
+def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
-def get_desktop_path():
-    username = os.getlogin()  # Get the current username
-    if platform.system() == "Windows":
-        return os.path.join("C:", "Users", username, "Desktop")
-    elif platform.system() == "Darwin":  # macOS is identified as 'Darwin'
-        return os.path.join("/Users", username, "Desktop")
-    elif platform.system() == "Linux":
-        return os.path.join("/home", username, "Desktop")
-    else:
-        raise Exception("Unsupported operating system")
-
-
-def get_wallpaper():
-    def get_wallpaper_windows():
-        SPI_GETDESKWALLPAPER = 0x73
-        MAX_PATH = 260
-        buffer = ctypes.create_unicode_buffer(MAX_PATH)
-        ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0)
-        return buffer.value
-
-    def get_wallpaper_macos():
-        script = """
-        tell application "System Events" to tell every desktop to get picture
     """
-        process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE)
+    Args:
-        output, error = process.communicate()
+        result (str): path to csv file
-        if error:
+        rules (Dict[str, List[Dict[str, str]]]): dict like
-            print("Error:", error)
+          {
-        else:
+            "expect": [{key: value}]
-            return output.strip().decode('utf-8')
+            "unexpect": [{key: value}]
+          }
 
-    def get_wallpaper_linux():
+    Returns:
-        try:
+        float
-            output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"])
+    """
-            return output.decode('utf-8').strip().replace('file://', '').replace("'", "")
-        except Exception as e:
-            print("Error:", e)
-            return None
-
-    os_name = platform.system()
-    if os_name == 'Windows':
-        return get_wallpaper_windows()
-    elif os_name == 'Darwin':
-        return get_wallpaper_macos()
-    elif os_name == 'Linux':
-        return get_wallpaper_linux()
-    else:
-        return "Unsupported OS"
 
+    expect_metrics = [False] * len(rules["except"])
+    unexpect_metric = True
+    with open(result) as f:
+        reader = csv.DictReader(f)
         
+        for rcd in reader:
+            for i, r in enumerate(rules["expect"]):
+                expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd)
+            unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules["unexpect"])
+    return float(all(expect_metrics) and unexpect_metric)

desktop_env/server/main.py

@@ -9,6 +9,7 @@ import pyautogui
 # from PIL import ImageGrab, Image
 from PIL import Image
 from flask import Flask, request, jsonify, send_file
+from typing import List
 
 app = Flask(__name__)
 
@@ -16,15 +17,16 @@ pyautogui.PAUSE = 0
 pyautogui.DARWIN_CATCH_UP_TIME = 0
 
 
+@app.route('/setup/execute', methods=['POST'])
 @app.route('/execute', methods=['POST'])
 def execute_command():
     data = request.json
     # The 'command' key in the JSON request should contain the command to be executed.
-    command = data.get('command', '')
+    command = data.get('command', [])
 
     # Execute the command without any safety checks.
     try:
-        result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
         return jsonify({
             'status': 'success',
             'output': result.stdout,
@@ -36,6 +38,21 @@ def execute_command():
             'message': str(e)
         }), 500
 
+@app.route('/setup/launch', methods=["POST"])
+def launch_app():
+    data = request.json
+    command: List[str] = data.get("command", [])
+
+    try:
+        subprocess.Popen(command)
+        return "{:} launched successfully".format(" ".join(command))
+    except Exception as e:
+        return jsonify( { "status": "error"
+                        , "message": str(e)
+                        }
+                      )\
+             , 500
+
 
 @app.route('/screenshot', methods=['GET'])
 def capture_screen_with_cursor():

evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json

@@ -3,10 +3,97 @@
   "snapshot": "thunderbird",
   "instruction": "Help me access my gmail account with address \"xx@gmail.com\" and password \"xxx\"",
   "source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird",
-  "config": [],
+  "config": [
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+            "path": "/home/david/thunderbird-profile.tar.gz"
+          }
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "tar",
+          "-xzv",
+          "--recursive-unlink",
+          "-f",
+          "/home/david/thunderbird-profile.tar.gz",
+          "-C",
+          "$HOME/"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "/usr/bin/thunderbird"
+        ]
+      }
+    }
+  ],
   "trajectory": "trajectories/",
   "related_apps": [
     "thunderbird"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "download",
+        "parameters": {
+          "files": [
+            {
+              "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
+              "path": "/home/david/firefox_decrypt.py"
+            }
+          ]
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "/home/david/firefox_decrypt.py",
+            "$HOME/.thunderbird",
+            "-n",
+            "-c",
+            "2",
+            "-f",
+            "csv",
+            "-d",
+            ","
+          ],
+          "stdout": "thunderbird-accounts.csv"
+        }
+      }
+    ],
+    "func": "check_csv",
+    "result": {
+      "type": "cache_file",
+      "path": "thunderbird-accounts.csv"
+    },
+    "expected": {
+      "rules": {
+        "expect": [
+          {
+            "url": "imap://imap.gmail.com",
+            "user": "xx@gmail.com",
+            "password": "xxx"
+          },
+          {
+            "url": "smtp://smtp.gmail.com",
+            "user": "xx@gmail.com",
+            "password": "xxx"
+          }
+        ]
+      }
+    }
+  }
 }

evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json

@@ -1,12 +1,93 @@
 {
   "id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
   "snapshot": "thunderbird",
-  "instruction": "Help remove the account \"xx@xx\"",
+  "instruction": "Help me to remove the account \"anonym-x2024@outlook.com\"",
   "source": "https://www.wikihow.com/Remove-an-Email-Account-from-Thunderbird",
-  "config": [],
+  "config": [
-  "trajectory": "trajectories/",
+    {
+      "type": "download",
+      "parameters": {
+        "files": [
+          {
+            "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+            "path": "/home/david/thunderbird-profile.tar.gz"
+          }
+        ]
+      }
+    },
+    {
+      "type": "execute",
+      "parameters": {
+        "command": [
+          "tar",
+          "-xzv",
+          "--recursive-unlink",
+          "-f",
+          "/home/david/thunderbird-profile.tar.gz",
+          "-C",
+          "$HOME/"
+        ]
+      }
+    },
+    {
+      "type": "launch",
+      "parameters": {
+        "command": [
+          "/usr/bin/thunderbird"
+        ]
+      }
+    }
+  ],
+  "trajectory": "trajectories/bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
   "related_apps": [
     "thunderbird"
   ],
-  "evaluator": "evaluation_dir"
+  "evaluator": {
+    "postconfig": [
+      {
+        "type": "download",
+        "parameters": {
+          "files": [
+            {
+              "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
+              "path": "/home/david/firefox_decrypt.py"
+            }
+          ]
+        }
+      },
+      {
+        "type": "execute",
+        "parameters": {
+          "command": [
+            "python3",
+            "/home/david/firefox_decrypt.py",
+            "$HOME/.thunderbird",
+            "-n",
+            "-c",
+            "2",
+            "-f",
+            "csv",
+            "-d",
+            ","
+          ],
+          "stdout": "thunderbird-accounts.csv"
+        }
+      }
+    ],
+    "func": "check_csv",
+    "result": {
+      "type": "cache_file",
+      "path": "thunderbird-accounts.csv"
+    },
+    "expected": {
+      "rules": {
+        "unexpect": [
+          {
+            "url": "imap://outlook.office365.com",
+            "user": "anonym-x2024@outlook.com"
+          }
+        ]
+      }
+    }
+  }
 }