diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index eea702f..1b3e581 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -21,6 +21,8 @@ class SetupController: # can add other setup steps def _download_setup(self, config): + if not config: + return if not 'download' in config: return for url, path in config['download']: @@ -43,6 +45,8 @@ class SetupController: print("An error occurred while trying to send the request:", e) def _change_wallpaper(self, config): + if not config: + return if not 'wallpaper' in config: return path = config['wallpaper'] @@ -68,6 +72,8 @@ class SetupController: raise NotImplementedError def _open_setup(self, config): + if not config: + return if not 'open' in config: return for path in config['open']: diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index d71aa2b..942c7c7 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -4,19 +4,25 @@ import os import subprocess import time import uuid +import platform from typing import List import gymnasium as gym + from desktop_env.controllers.python import PythonController from desktop_env.controllers.setup import SetupController def _execute_command(command: List[str]) -> None: - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, text=True) - if result.returncode != 0: - raise Exception("\033[91m" + result.stdout + result.stderr + "\033[0m") - return result.stdout + if command[:4] == ["vmrun", "-T", "ws", "start"]: + p = subprocess.Popen(command) + p.wait() + else: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, text=True) + if result.returncode != 0: + raise Exception("\033[91m" + result.stdout + result.stderr + "\033[0m") + return result.stdout class DesktopEnv(gym.Env): @@ -27,7 +33,7 @@ class DesktopEnv(gym.Env): path_to_vm: str, snapshot_path: str = "base", config: dict = None, - action_space: str = "pyautogui", + action_space: str = "computer_13", ): # Initialize environment variables self.path_to_vm = path_to_vm @@ -111,6 +117,7 @@ class DesktopEnv(gym.Env): return observation def step(self, action, pause=0.5): + # fixme: add reminding logic here, decide if the action is valid for the current action_space if self.action_space == "computer_13": # the set of all possible actions defined in the action representation self.controller.execute_action(action) diff --git a/main.py b/main.py index a52ad90..3c8acce 100644 --- a/main.py +++ b/main.py @@ -14,242 +14,61 @@ def human_agent(): # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""", path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""", # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", - snapshot_path="base_setup", - config=example["config"], + action_space="computer_13", + snapshot_path="base_setup2", + # config=example["config"], ) # reset the environment to certain snapshot observation = env.reset() done = False - trajectory = [{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, - {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}, - {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, - {"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, - {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, - {"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, - {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, - {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, - {"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, - {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, - {"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, - {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, - {"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, - {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, - {"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, - {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, - {"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, - {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, - {"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 692, "y": 459}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 690, "y": 459}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 675, "y": 460}}, - {"action_type": "KEY_UP", "parameters": {"key": "s"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 674, "y": 460}}, - {"action_type": "KEY_UP", "parameters": {"key": "f"}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482 - }}, - {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, - {"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, - {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, - {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, - {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, - {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 568, "y": 509}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 511}}, - {"action_type": "KEY_UP", "parameters": {"key": "d"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 613, "y": 516}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 616, "y": 513}}, - {"action_type": "KEY_UP", "parameters": {"key": "s"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 341}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "d"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 342}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 350}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 352}}, - {"action_type": "KEY_UP", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 602, "y": 355}}, - {"action_type": "KEY_UP", "parameters": {"key": "d"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 617, "y": 453}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 610, "y": 489}}, - {"action_type": "KEY_UP", "parameters": {"key": "s"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, - {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, - {"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 538, "y": 588}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "s"}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 590}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 591}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 544, "y": 595}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 545, "y": 597}}, - {"action_type": "KEY_UP", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 536, "y": 605}}, - {"action_type": "KEY_DOWN", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 526, "y": 606}}, - {"action_type": "KEY_UP", "parameters": {"key": "g"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 513, "y": 606}}, - {"action_type": "KEY_UP", "parameters": {"key": "f"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 490, "y": 604}}, - {"action_type": "KEY_UP", "parameters": {"key": "s"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 489, "y": 604}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "right"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 458}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 559, "y": 458}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, - {"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 553, "y": 375}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 504}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, - {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, - {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, - {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, - {"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 3}}, - {"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}}, - {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, - {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, - {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, - {"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 755, "y": 1057}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 754, "y": 1057}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, - {"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}}, - {"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}}, - {"action_type": "MOUSE_UP", "parameters": {"button": "left"}}] + trajectory = [ + { + "action_type": "MOVE_TO", + "parameters": { + "x": 754, + "y": 1057 + } + }, + {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}, + { + "action_type": "MOVE_TO", + "parameters": { + "x": 754, + "y": 1057 + } + }, + { + "action_type": "MOVE_TO", + "parameters": { + "x": 754, + "y": 1057 + } + }, + { + "action_type": "MOVE_TO", + "parameters": { + "x": 754, + "y": 1057 + } + }, + { + "action_type": "MOVE_TO", + "parameters": { + "x": 754, + "y": 1057 + } + }, + { + "action_type": "MOVE_TO", + "parameters": { + "x": 754, + "y": 1057 + } + } + + ] for i in range(len(trajectory)): # action = get_human_action() @@ -260,7 +79,7 @@ def human_agent(): # } print(trajectory[i]) - observation, reward, done, info = env.step(trajectory[i], pause=1) + observation, reward, done, info = env.step(trajectory[i], pause=5) print("Observation:", observation) print("Reward:", reward) print("Info:", info)