diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py new file mode 100644 index 0000000..d2abc97 --- /dev/null +++ b/desktop_env/controllers/setup.py @@ -0,0 +1,40 @@ +import requests +import json + +class SetupController: + def __init__(self, http_server: str): + self.http_server = http_server + "/setup" + + def setup(self, config): + """ + Setup Config: + { + download: list[tuple[string]], # a list of tuples of url of file to download and the save path + ... + } + """ + self._download_setup(config) + # can add other setup steps + + + def _download_setup(self, config): + if not 'download' in config: + return + for url, path in config['download']: + if not url or not path: + raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") + + payload = json.dumps({"url": url, "path": path}) + headers = { + 'Content-Type': 'application/json' + } + + # send request to server to download file + try: + response = requests.post(self.http_server + "/download_file", headers=headers, data=payload) + if response.status_code == 200: + print("Command executed successfully:", response.text) + else: + print("Failed to download file. Status code:", response.text) + except requests.exceptions.RequestException as e: + print("An error occurred while trying to send the request:", e) diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index b4347cb..99a9a26 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -9,12 +9,14 @@ from typing import List import gymnasium as gym from desktop_env.controllers.python import PythonController +from desktop_env.controllers.setup import SetupController def _execute_command(command: List[str]) -> None: result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, text=True) if result.returncode != 0: raise Exception("\033[91m" + result.stdout + result.stderr + "\033[0m") + return result.stdout class DesktopEnv(gym.Env): @@ -23,19 +25,19 @@ class DesktopEnv(gym.Env): def __init__( self, path_to_vm: str, - host: str = "192.168.7.128:5000", snapshot_path: str = "base", action_space: str = "pyautogui", ): # Initialize environment variables self.path_to_vm = path_to_vm - self.host = host self.snapshot_path = snapshot_path # todo: handling the logic of snapshot directory # Initialize emulator and controller print("Initializing...") self._start_emulator() + self.host = f"http://{self._get_vm_ip()}:5000" self.controller = PythonController(http_server=self.host) + self.setup_controller = SetupController(http_server=self.host) # mode: human or machine assert action_space in ["computer_13", "pyautogui"] @@ -53,10 +55,23 @@ class DesktopEnv(gym.Env): else: print("Starting VM...") _execute_command(["vmrun", "-T", "ws", "start", self.path_to_vm]) - time.sleep(10) + time.sleep(3) except subprocess.CalledProcessError as e: print(f"Error executing command: {e.output.decode().strip()}") + def _get_vm_ip(self): + max_retries = 10 + print("Getting IP Address...") + for _ in range(max_retries): + try: + output = _execute_command(["vmrun", "-T", "ws", "getGuestIPAddress", self.path_to_vm]).strip() + print(f"IP address: {output}") + return output + except: + time.sleep(5) + print("Retrying...") + raise Exception("Failed to get VM IP address!") + def _save_state(self): _execute_command(["vmrun", "-T", "ws" "snapshot", self.path_to_vm, self.snapshot_path]) @@ -105,6 +120,9 @@ class DesktopEnv(gym.Env): done = False # todo: Define episode termination condition for each example info = {} return observation, reward, done, info + + def setup(self, config: dict): + self.setup_controller.setup(config) def render(self, mode='rgb_array'): if mode == 'rgb_array': diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index f9e8dcd..93884ca 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import platform import subprocess import requests @@ -83,6 +84,34 @@ def get_platform(): def get_cursor_position(): return pyautogui.position().x, pyautogui.position().y +@app.route("/setup/download_file", methods=['POST']) +def download_file(): + data = request.json + url = data.get('url', None) + path = data.get('path', None) + + if not url or not path: + return "Path or URL not supplied!", 400 + + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + max_retries = 3 + for i in range(max_retries): + try: + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + return "File downloaded successfully" + + except requests.RequestException as e: + print(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)") + + return f"Failed to download {url}. No retries left. Error: {e}", 500 if __name__ == '__main__': app.run(debug=True, host="0.0.0.0") diff --git a/main.py b/main.py index 6cff55d..f587ddc 100644 --- a/main.py +++ b/main.py @@ -8,20 +8,16 @@ def human_agent(): env = DesktopEnv( path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""", # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", - # host="192.168.7.128", - host="http://192.168.13.128:5000", - snapshot_path="libreoffice_calc", - action_space="computer_13", + snapshot_path="base3", ) + # example setup + env.setup({"download": [("https://images.unsplash.com/photo-1683009427051-00a2fe827a2c", "C:/Users/Yuri/Desktop/1.jpg")]}) + # reset the environment to certain snapshot observation = env.reset() done = False - # trajectory = [ - # "pyautogui.moveTo(10, 100)", - # "pyautogui.click(button='right')" - # ] trajectory = [{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 692, "y": 459}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 690, "y": 459}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 675, "y": 460}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 674, "y": 460}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 568, "y": 509}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 511}}, {"action_type": "KEY_UP", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 613, "y": 516}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 616, "y": 513}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 341}}, {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 342}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 350}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 352}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 602, "y": 355}}, {"action_type": "KEY_UP", "parameters": {"key": "d"}}, {"action_type": "MOVE_TO", "parameters": {"x": 617, "y": 453}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 610, "y": 489}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 538, "y": 588}}, {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 590}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 591}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 544, "y": 595}}, {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 545, "y": 597}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 536, "y": 605}}, {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 526, "y": 606}}, {"action_type": "KEY_UP", "parameters": {"key": "g"}}, {"action_type": "MOVE_TO", "parameters": {"x": 513, "y": 606}}, {"action_type": "KEY_UP", "parameters": {"key": "f"}}, {"action_type": "MOVE_TO", "parameters": {"x": 490, "y": 604}}, {"action_type": "KEY_UP", "parameters": {"key": "s"}}, {"action_type": "MOVE_TO", "parameters": {"x": 489, "y": 604}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 458}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 559, "y": 458}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 553, "y": 375}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 504}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 3}}, {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 755, "y": 1057}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 754, "y": 1057}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}]