diff --git a/.gitignore b/.gitignore index b3945ef..3fb2a90 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,9 @@ instructor-embedding # frontend cache frontend/node_modules/ frontend/.next/ -frontend/.idea \ No newline at end of file +frontend/.idea + +tags +snapshots +branch_flag +branch-config diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 1b3e581..a550389 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -1,6 +1,8 @@ import requests import json +from typing import Dict, List +from typing import Any class SetupController: def __init__(self, http_server: str): @@ -14,18 +16,42 @@ class SetupController: ... } """ - self._download_setup(config) - self._change_wallpaper(config) + + for cfg in config: + config_type: str = cfg["type"] + parameters: Dict[str, Any] = cfg["parameters"] + + # Assumes all the setup the functions should follow this name + # protocol + setup_function: str = "_{:}_setup".format(config_type) + assert hasattr(self, setup_function) + getattr(self, setup_function)(**parameters) + + #self._download_setup(config) + #self._change_wallpaper(config) # self._tidy_desktop(config) todo: implement this - self._open_setup(config) + #self._open_setup(config) # can add other setup steps - def _download_setup(self, config): - if not config: - return - if not 'download' in config: - return - for url, path in config['download']: + def _download_setup(self, files: List[Dict[str, str]]): + """ + Args: + files (List[Dict[str, str]]): files to download. lisf of dict like + { + "url": str, the url to download + "path": str, the path on the VM to store the downloaded file + } + """ + + #if not config: + #return + #if not 'download' in config: + #return + #for url, path in config['download']: + for f in files: + url: str = f["url"] + path: str = f["path"] + if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") @@ -44,12 +70,13 @@ class SetupController: except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) - def _change_wallpaper(self, config): - if not config: - return - if not 'wallpaper' in config: - return - path = config['wallpaper'] + def _change_wallpaper_setup(self, path: str): + #if not config: + #return + #if not 'wallpaper' in config: + #return + + #path = config['wallpaper'] if not path: raise Exception(f"Setup Wallpaper - Invalid path ({path}).") @@ -68,29 +95,29 @@ class SetupController: except requests.exceptions.RequestException as e: print("An error occurred while trying to send the request:", e) - def _tidy_desktop(self, config): + def _tidy_desktop_setup(self, **config): raise NotImplementedError - def _open_setup(self, config): - if not config: - return - if not 'open' in config: - return - for path in config['open']: - if not path: - raise Exception(f"Setup Open - Invalid path ({path}).") + def _open_setup(self, path: str): + #if not config: + #return + #if not 'open' in config: + #return + #for path in config['open']: + if not path: + raise Exception(f"Setup Open - Invalid path ({path}).") - payload = json.dumps({"path": path}) - headers = { - 'Content-Type': 'application/json' - } + payload = json.dumps({"path": path}) + headers = { + 'Content-Type': 'application/json' + } - # send request to server to open file - try: - response = requests.post(self.http_server + "/open_file", headers=headers, data=payload) - if response.status_code == 200: - print("Command executed successfully:", response.text) - else: - print("Failed to open file. Status code:", response.text) - except requests.exceptions.RequestException as e: - print("An error occurred while trying to send the request:", e) + # send request to server to open file + try: + response = requests.post(self.http_server + "/open_file", headers=headers, data=payload) + if response.status_code == 200: + print("Command executed successfully:", response.text) + else: + print("Failed to open file. Status code:", response.text) + except requests.exceptions.RequestException as e: + print("An error occurred while trying to send the request:", e) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 183411d..34270c8 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -60,7 +60,7 @@ class DesktopEnv(gym.Env): def _start_emulator(self): while True: try: - output = subprocess.check_output(f"vmrun -T ws list", shell=True, stderr=subprocess.STDOUT) + output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT) output = output.decode() if self.path_to_vm.lstrip("~/") in output: print("VM is running.") diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 245054e..9f3c5a4 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -4,9 +4,10 @@ import platform import subprocess import requests -import Xlib.display +#import Xlib.display import pyautogui -from PIL import ImageGrab, Image +#from PIL import ImageGrab, Image +from PIL import Image from flask import Flask, request, jsonify, send_file app = Flask(__name__) @@ -180,7 +181,11 @@ def open_file(): return f"File not found: {path}", 404 try: - os.startfile(path) + if platform.system()=="Windows": + os.startfile(path) + else: + open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open" + subprocess.Popen([open_cmd, str(path)]) return "File opened successfully" except Exception as e: return f"Failed to open {path}. Error: {e}", 500 diff --git a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index 5a01d56..b3f61c2 100644 --- a/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -3,17 +3,25 @@ "snapshot": "libreoffice_calc", "instruction": "Fill the missing row and column which show the total value", "source": "https://youtube.com/shorts/feldd-Pn48c?si=9xJiem2uAHm6Jshb", - "config": { - "download": [ - [ - "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851", - "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" - ] - }, + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851", + "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx" + } + } + ], "trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850", "related_apps": [ "libreoffice calc" diff --git a/main.py b/main.py index e8658a7..1c6a242 100644 --- a/main.py +++ b/main.py @@ -7,19 +7,16 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json", "r") as f: + with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f: example = json.load(f) - env = DesktopEnv( - # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""", - path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx""", - # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", - action_space="computer_13", - snapshot_path="base_setup", - instruction=example["instruction"], - # config=example["config"], - # evaluator=example["evaluator"] - ) + env = DesktopEnv( path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx" + , action_space="computer_13" + , snapshot_path="base_setup" + , instruction=example["instruction"] + , config=example["config"] + , evaluator=example["evaluator"] + ) # reset the environment to certain snapshot observation = env.reset()