diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index e28287b..3c28d3f 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -6,7 +6,7 @@ import uuid import os.path from typing import Dict, List -from typing import Any +from typing import Any, Union class SetupController: @@ -145,7 +145,7 @@ class SetupController: print("An error occurred while trying to send the request:", e) def _tidy_desktop_setup(self, **config): - raise NotImplementedError + raise NotImplementedError() def _open_setup(self, path: str): # if not config: @@ -170,3 +170,56 @@ class SetupController: print("Failed to open file. Status code:", response.text) except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) + + def _launch_setup(self, command: List[str]): + if not command: + raise Exception("Empty comman to launch.") + + payload = json.dumps({"command": command}) + headers = {"Content-Type": "application/json"} + + try: + response = requests.post(self.http_server + "/launch", headers=headers, data=payload) + if response.status_code == 200: + print("Command executed successfully:", response.text) + else: + print("Failed to launch application. Status code:", response.text) + except requests.exceptions.RequestException as e: + print("An error occurred while trying to send the request:", e) + + def _execute_setup(self, command: List[str], stdout: str = "", stderr: str = ""): + if not command: + raise Exception("Empty comman to launch.") + + payload = json.dumps({"command": command}) + headers = {"Content-Type": "application/json"} + + try: + response = requests.post(self.http_server + "/launch", headers=headers, data=payload) + if response.status_code == 200: + results: Dict[str, str] = response.json() + if stdout: + with open(os.path.join(self.cache_dir, stdout), "w") as f: + f.write(results["output"]) + if stderr: + with open(os.path.join(self.cache_dir, stderr), "w") as f: + f.write(results["error"]) + print( "Command executed successfully: {:} ->".format(" ".join(command)) + , response.text + ) + else: + print("Failed to launch application. Status code:", response.text) + except requests.exceptions.RequestException as e: + print("An error occurred while trying to send the request:", e) + + def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]): + # TODO + raise NotImplementedError() + def _replay_setup(self, trajectory: str): + """ + Args: + trajectory (str): path to the replay trajectory file + """ + + # TODO + raise NotImplementedError() diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 03086ff..1845f94 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -68,19 +68,8 @@ class DesktopEnv(gym.Env): self.cache_dir_base: str = cache_dir # task-aware stuffs - self.snapshot_path = task_config["snapshot"] # todo: handling the logic of snapshot directory - self.task_id: str = task_config["id"] - self.cache_dir: str = os.path.join(self.cache_dir_base, self.task_id) - os.makedirs(self.cache_dir, exist_ok=True) - self.instruction = task_config["instruction"] - self.config = task_config["config"] - - self.evaluator = task_config["evaluator"] - self.metric: Metric = getattr(metrics, self.evaluator["func"]) - self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) - self.expected_getter: Getter = getattr(getters, "get_{:}".format( - self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None - self.metric_options: Dict[str, Any] = self.evaluator.get("options", {}) + # todo: handling the logic of snapshot directory + self._set_task_info(task_config) # Initialize emulator and controller print("Initializing...") @@ -151,25 +140,27 @@ class DesktopEnv(gym.Env): screenshot_image_path = self._get_screenshot() return screenshot_image_path + def _set_task_info(self, task_config: Dict[str, Any]): + self.snapshot_path = task_config["snapshot"] + self.task_id: str = task_config["id"] + self.cache_dir: str = os.path.join(self.cache_dir_base, self.task_id) + os.makedirs(self.cache_dir, exist_ok=True) + self.instruction = task_config["instruction"] + self.config = task_config["config"] + + self.evaluator = task_config["evaluator"] + self.metric: Metric = getattr(metrics, self.evaluator["func"]) + self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) + self.expected_getter: Getter = getattr(getters, "get_{:}".format( + self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None + self.metric_options: Dict[str, Any] = self.evaluator.get("options", {}) + def reset(self, task_config: Optional[Dict[str, Any]] = None, seed=None, options=None): print("Resetting environment...") print("Switching task...") if task_config is not None: - self.snapshot_path = task_config["snapshot"] - self.task_id = task_config["id"] - self.cache_dir = os.path.join(self.cache_dir_base, self.task_id) - os.makedirs(self.cache_dir, exist_ok=True) - self.instruction = task_config["instruction"] - self.config = task_config["config"] - - self.evaluator = task_config["evaluator"] - self.metric: Metric = getattr(metrics, self.evaluator["func"]) - self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"])) - self.expected_getter: Getter = getattr(getters, "get_{:}".format( - self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None - self.metric_options = self.evaluator.get("options", {}) - + self._set_task_info(task_config) self.setup_controller.reset_cache_dir(self.cache_dir) print("Setting counters...") @@ -228,6 +219,9 @@ class DesktopEnv(gym.Env): """ Evaluate whether the task is successfully completed. """ + + self.setup_controller.setup(self.evaluator["postconfig"]) + result_state = self.result_getter(self, self.evaluator["result"]) expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \ else None diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 81a23fd..32ce34b 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,2 +1,2 @@ -from .file import get_cloud_file, get_vm_file +from .file import get_cloud_file, get_vm_file, get_cache_file from .misc import get_rule diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index a9be430..bc61bec 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -33,8 +33,6 @@ def get_vm_file(env, config: Dict[str, str]) -> str: """ _path = os.path.join(env.cache_dir, config["dest"]) - if os.path.exists(_path): - return _path file = env.controller.get_file(config["path"]) with open(_path, "wb") as f: @@ -42,3 +40,12 @@ def get_vm_file(env, config: Dict[str, str]) -> str: return _path +def get_cache_file(env, config: Dict[str, str]) -> str: + """ + Config: + path (str): relative path in cache dir + """ + + _path = os.path.join(env.cache_dir, config["path"]) + assert os.path.exists(_path) + return _path diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index 489d00c..4f82666 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -1,8 +1,63 @@ from typing import TypeVar +import platform +import subprocess +import ctypes +import os + R = TypeVar("Rule") def get_rule(env, config: R) -> R: """ Returns the rule as-is. """ return config["rules"] + + +def get_desktop_path(): + username = os.getlogin() # Get the current username + if platform.system() == "Windows": + return os.path.join("C:", "Users", username, "Desktop") + elif platform.system() == "Darwin": # macOS is identified as 'Darwin' + return os.path.join("/Users", username, "Desktop") + elif platform.system() == "Linux": + return os.path.join("/home", username, "Desktop") + else: + raise Exception("Unsupported operating system") + + +def get_wallpaper(): + def get_wallpaper_windows(): + SPI_GETDESKWALLPAPER = 0x73 + MAX_PATH = 260 + buffer = ctypes.create_unicode_buffer(MAX_PATH) + ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0) + return buffer.value + + def get_wallpaper_macos(): + script = """ + tell application "System Events" to tell every desktop to get picture + """ + process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE) + output, error = process.communicate() + if error: + print("Error:", error) + else: + return output.strip().decode('utf-8') + + def get_wallpaper_linux(): + try: + output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"]) + return output.decode('utf-8').strip().replace('file://', '').replace("'", "") + except Exception as e: + print("Error:", e) + return None + + os_name = platform.system() + if os_name == 'Windows': + return get_wallpaper_windows() + elif os_name == 'Darwin': + return get_wallpaper_macos() + elif os_name == 'Linux': + return get_wallpaper_linux() + else: + return "Unsupported OS" diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 159b3d5..a24e8e3 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -5,3 +5,4 @@ from .docs import compare_font_names, compare_subscript_contains, has_page_numbe from .docs import is_first_line_centered, check_file_exists, compare_contains_image from .pdf import check_pdf_pages from .libreoffice import check_libre_locale +from .general import check_csv diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 460b8e8..d0776eb 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -1,57 +1,30 @@ -import platform -import subprocess -import ctypes -import os +import csv +from typing import Dict, List +def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float: + return all(k in item and item[k]==val for k, val in pattern.items()) -# todo: move to getter module -def get_desktop_path(): - username = os.getlogin() # Get the current username - if platform.system() == "Windows": - return os.path.join("C:", "Users", username, "Desktop") - elif platform.system() == "Darwin": # macOS is identified as 'Darwin' - return os.path.join("/Users", username, "Desktop") - elif platform.system() == "Linux": - return os.path.join("/home", username, "Desktop") - else: - raise Exception("Unsupported operating system") - - -def get_wallpaper(): - def get_wallpaper_windows(): - SPI_GETDESKWALLPAPER = 0x73 - MAX_PATH = 260 - buffer = ctypes.create_unicode_buffer(MAX_PATH) - ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0) - return buffer.value - - def get_wallpaper_macos(): - script = """ - tell application "System Events" to tell every desktop to get picture - """ - process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE) - output, error = process.communicate() - if error: - print("Error:", error) - else: - return output.strip().decode('utf-8') - - def get_wallpaper_linux(): - try: - output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"]) - return output.decode('utf-8').strip().replace('file://', '').replace("'", "") - except Exception as e: - print("Error:", e) - return None - - os_name = platform.system() - if os_name == 'Windows': - return get_wallpaper_windows() - elif os_name == 'Darwin': - return get_wallpaper_macos() - elif os_name == 'Linux': - return get_wallpaper_linux() - else: - return "Unsupported OS" +def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: + """ + Args: + result (str): path to csv file + rules (Dict[str, List[Dict[str, str]]]): dict like + { + "expect": [{key: value}] + "unexpect": [{key: value}] + } + Returns: + float + """ + expect_metrics = [False] * len(rules["except"]) + unexpect_metric = True + with open(result) as f: + reader = csv.DictReader(f) + + for rcd in reader: + for i, r in enumerate(rules["expect"]): + expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd) + unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules["unexpect"]) + return float(all(expect_metrics) and unexpect_metric) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index f6ba06f..e565787 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -9,6 +9,7 @@ import pyautogui # from PIL import ImageGrab, Image from PIL import Image from flask import Flask, request, jsonify, send_file +from typing import List app = Flask(__name__) @@ -16,15 +17,16 @@ pyautogui.PAUSE = 0 pyautogui.DARWIN_CATCH_UP_TIME = 0 +@app.route('/setup/execute', methods=['POST']) @app.route('/execute', methods=['POST']) def execute_command(): data = request.json # The 'command' key in the JSON request should contain the command to be executed. - command = data.get('command', '') + command = data.get('command', []) # Execute the command without any safety checks. try: - result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) return jsonify({ 'status': 'success', 'output': result.stdout, @@ -36,6 +38,21 @@ def execute_command(): 'message': str(e) }), 500 +@app.route('/setup/launch', methods=["POST"]) +def launch_app(): + data = request.json + command: List[str] = data.get("command", []) + + try: + subprocess.Popen(command) + return "{:} launched successfully".format(" ".join(command)) + except Exception as e: + return jsonify( { "status": "error" + , "message": str(e) + } + )\ + , 500 + @app.route('/screenshot', methods=['GET']) def capture_screen_with_cursor(): diff --git a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json index c3a8a39..21adb0d 100644 --- a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json +++ b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json @@ -3,10 +3,97 @@ "snapshot": "thunderbird", "instruction": "Help me access my gmail account with address \"xx@gmail.com\" and password \"xxx\"", "source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/david/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/david/thunderbird-profile.tar.gz", + "-C", + "$HOME/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "thunderbird" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", + "path": "/home/david/firefox_decrypt.py" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "/home/david/firefox_decrypt.py", + "$HOME/.thunderbird", + "-n", + "-c", + "2", + "-f", + "csv", + "-d", + "," + ], + "stdout": "thunderbird-accounts.csv" + } + } + ], + "func": "check_csv", + "result": { + "type": "cache_file", + "path": "thunderbird-accounts.csv" + }, + "expected": { + "rules": { + "expect": [ + { + "url": "imap://imap.gmail.com", + "user": "xx@gmail.com", + "password": "xxx" + }, + { + "url": "smtp://smtp.gmail.com", + "user": "xx@gmail.com", + "password": "xxx" + } + ] + } + } + } } diff --git a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json index a44db87..e659ede 100644 --- a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json +++ b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json @@ -1,12 +1,93 @@ { "id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4", "snapshot": "thunderbird", - "instruction": "Help remove the account \"xx@xx\"", + "instruction": "Help me to remove the account \"anonym-x2024@outlook.com\"", "source": "https://www.wikihow.com/Remove-an-Email-Account-from-Thunderbird", - "config": [], - "trajectory": "trajectories/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/david/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/david/thunderbird-profile.tar.gz", + "-C", + "$HOME/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/bb5e4c0d-f964-439c-97b6-bdb9747de3f4", "related_apps": [ "thunderbird" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", + "path": "/home/david/firefox_decrypt.py" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "/home/david/firefox_decrypt.py", + "$HOME/.thunderbird", + "-n", + "-c", + "2", + "-f", + "csv", + "-d", + "," + ], + "stdout": "thunderbird-accounts.csv" + } + } + ], + "func": "check_csv", + "result": { + "type": "cache_file", + "path": "thunderbird-accounts.csv" + }, + "expected": { + "rules": { + "unexpect": [ + { + "url": "imap://outlook.office365.com", + "user": "anonym-x2024@outlook.com" + } + ] + } + } + } }