lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix action_space setup

Browse Source
This commit is contained in:
Timothyxxx
2023-12-06 22:59:19 +08:00
parent b9c317f0f5
commit 343b40ecac
3 changed files with 68 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
desktop_env/controllers/setup.py
View File

@@ -21,6 +21,8 @@ class SetupController:
# can add other setup steps # can add other setup steps
def _download_setup(self, config): def _download_setup(self, config):
if not config:
return
if not 'download' in config: if not 'download' in config:
return return
for url, path in config['download']: for url, path in config['download']:
@@ -43,6 +45,8 @@ class SetupController:
print("An error occurred while trying to send the request:", e) print("An error occurred while trying to send the request:", e)
def _change_wallpaper(self, config): def _change_wallpaper(self, config):
if not config:
return
if not 'wallpaper' in config: if not 'wallpaper' in config:
return return
path = config['wallpaper'] path = config['wallpaper']
@@ -68,6 +72,8 @@ class SetupController:
raise NotImplementedError raise NotImplementedError
def _open_setup(self, config): def _open_setup(self, config):
if not config:
return
if not 'open' in config: if not 'open' in config:
return return
for path in config['open']: for path in config['open']:

17
desktop_env/envs/desktop_env.py
View File

@@ -4,19 +4,25 @@ import os
import subprocess import subprocess
import time import time
import uuid import uuid
import platform
from typing import List from typing import List
import gymnasium as gym import gymnasium as gym
from desktop_env.controllers.python import PythonController from desktop_env.controllers.python import PythonController
from desktop_env.controllers.setup import SetupController from desktop_env.controllers.setup import SetupController
def _execute_command(command: List[str]) -> None: def _execute_command(command: List[str]) -> None:
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, text=True) if command[:4] == ["vmrun", "-T", "ws", "start"]:
if result.returncode != 0: p = subprocess.Popen(command)
raise Exception("\033[91m" + result.stdout + result.stderr + "\033[0m") p.wait()
return result.stdout else:
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60, text=True)
if result.returncode != 0:
raise Exception("\033[91m" + result.stdout + result.stderr + "\033[0m")
return result.stdout
class DesktopEnv(gym.Env): class DesktopEnv(gym.Env):
@@ -27,7 +33,7 @@ class DesktopEnv(gym.Env):
path_to_vm: str, path_to_vm: str,
snapshot_path: str = "base", snapshot_path: str = "base",
config: dict = None, config: dict = None,
action_space: str = "pyautogui", action_space: str = "computer_13",
): ):
# Initialize environment variables # Initialize environment variables
self.path_to_vm = path_to_vm self.path_to_vm = path_to_vm
@@ -111,6 +117,7 @@ class DesktopEnv(gym.Env):
return observation return observation
def step(self, action, pause=0.5): def step(self, action, pause=0.5):
# fixme: add reminding logic here, decide if the action is valid for the current action_space
if self.action_space == "computer_13": if self.action_space == "computer_13":
# the set of all possible actions defined in the action representation # the set of all possible actions defined in the action representation
self.controller.execute_action(action) self.controller.execute_action(action)

281
main.py
View File

@@ -14,242 +14,61 @@ def human_agent():
# path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""", # path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""",
path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""", path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Win10\Win10.vmx""",
# path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx", # path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
snapshot_path="base_setup", action_space="computer_13",
config=example["config"], snapshot_path="base_setup2",
# config=example["config"],
) )
# reset the environment to certain snapshot # reset the environment to certain snapshot
observation = env.reset() observation = env.reset()
done = False done = False
trajectory = [{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, trajectory = [
{"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}, {
{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, "action_type": "MOVE_TO",
{"action_type": "MOVE_TO", "parameters": {"x": 501, "y": 382}}, "parameters": {
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, "x": 754,
{"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, "y": 1057
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, }
{"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, },
{"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}},
{"action_type": "MOVE_TO", "parameters": {"x": 560, "y": 385}}, {
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, "action_type": "MOVE_TO",
{"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, "parameters": {
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, "x": 754,
{"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, "y": 1057
{"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, }
{"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 386}}, },
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, {
{"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, "action_type": "MOVE_TO",
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, "parameters": {
{"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, "x": 754,
{"action_type": "MOVE_TO", "parameters": {"x": 716, "y": 389}}, "y": 1057
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, }
{"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, },
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, {
{"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, "action_type": "MOVE_TO",
{"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, "parameters": {
{"action_type": "MOVE_TO", "parameters": {"x": 639, "y": 409}}, "x": 754,
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, "y": 1057
{"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, }
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, },
{"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, {
{"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, "action_type": "MOVE_TO",
{"action_type": "MOVE_TO", "parameters": {"x": 711, "y": 409}}, "parameters": {
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, "x": 754,
{"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, "y": 1057
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, }
{"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 448}}, },
{"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, {
{"action_type": "MOVE_TO", "parameters": {"x": 644, "y": 449}}, "action_type": "MOVE_TO",
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, "parameters": {
{"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, "x": 754,
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}}, "y": 1057
{"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, }
{"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}}, }
{"action_type": "MOVE_TO", "parameters": {"x": 720, "y": 446}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}, ]
{"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}},
{"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}},
{"action_type": "MOVE_TO", "parameters": {"x": 660, "y": 458}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 692, "y": 459}},
{"action_type": "KEY_DOWN", "parameters": {"key": "s"}},
{"action_type": "MOVE_TO", "parameters": {"x": 690, "y": 459}},
{"action_type": "KEY_DOWN", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 675, "y": 460}},
{"action_type": "KEY_UP", "parameters": {"key": "s"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 674, "y": 460}},
{"action_type": "KEY_UP", "parameters": {"key": "f"}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 482
}},
{"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}},
{"action_type": "MOVE_TO", "parameters": {"x": 637, "y": 483}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}},
{"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}},
{"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 501}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}},
{"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}},
{"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 501}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 568, "y": 509}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "s"}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "d"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 511}},
{"action_type": "KEY_UP", "parameters": {"key": "d"}},
{"action_type": "MOVE_TO", "parameters": {"x": 613, "y": 516}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 616, "y": 513}},
{"action_type": "KEY_UP", "parameters": {"key": "s"}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 337}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "s"}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 341}},
{"action_type": "KEY_DOWN", "parameters": {"key": "d"}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 342}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 600, "y": 350}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 601, "y": 352}},
{"action_type": "KEY_UP", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 602, "y": 355}},
{"action_type": "KEY_UP", "parameters": {"key": "d"}},
{"action_type": "MOVE_TO", "parameters": {"x": 617, "y": 453}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 610, "y": 489}},
{"action_type": "KEY_UP", "parameters": {"key": "s"}},
{"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}},
{"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}},
{"action_type": "MOVE_TO", "parameters": {"x": 533, "y": 586}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 538, "y": 588}},
{"action_type": "KEY_DOWN", "parameters": {"key": "s"}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 590}},
{"action_type": "KEY_DOWN", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 541, "y": 591}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 544, "y": 595}},
{"action_type": "KEY_DOWN", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 545, "y": 597}},
{"action_type": "KEY_UP", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 536, "y": 605}},
{"action_type": "KEY_DOWN", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 526, "y": 606}},
{"action_type": "KEY_UP", "parameters": {"key": "g"}},
{"action_type": "MOVE_TO", "parameters": {"x": 513, "y": 606}},
{"action_type": "KEY_UP", "parameters": {"key": "f"}},
{"action_type": "MOVE_TO", "parameters": {"x": 490, "y": 604}},
{"action_type": "KEY_UP", "parameters": {"key": "s"}},
{"action_type": "MOVE_TO", "parameters": {"x": 489, "y": 604}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 451, "y": 604}},
{"action_type": "MOUSE_UP", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 563, "y": 590}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 579}},
{"action_type": "MOUSE_UP", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 615, "y": 612}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 623, "y": 458}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 626, "y": 455}},
{"action_type": "MOUSE_UP", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 456}},
{"action_type": "MOUSE_UP", "parameters": {"button": "right"}},
{"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 458}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 559, "y": 458}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 557, "y": 459}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}},
{"action_type": "MOVE_TO", "parameters": {"x": 570, "y": 427}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 628, "y": 423}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 553, "y": 375}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 647, "y": 504}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 649, "y": 504}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 2}},
{"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}},
{"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}},
{"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 371}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}},
{"action_type": "CLICK", "parameters": {"button": "left", "num_clicks": 3}},
{"action_type": "MOVE_TO", "parameters": {"x": 564, "y": 335}},
{"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}},
{"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}},
{"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}},
{"action_type": "MOVE_TO", "parameters": {"x": 567, "y": 333}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 573, "y": 332}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 618, "y": 411}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 755, "y": 1057}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 754, "y": 1057}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}},
{"action_type": "MOUSE_DOWN", "parameters": {"button": "left"}},
{"action_type": "MOVE_TO", "parameters": {"x": 701, "y": 295}},
{"action_type": "MOUSE_UP", "parameters": {"button": "left"}}]
for i in range(len(trajectory)): for i in range(len(trajectory)):
# action = get_human_action() # action = get_human_action()
@@ -260,7 +79,7 @@ def human_agent():
# } # }
print(trajectory[i]) print(trajectory[i])
observation, reward, done, info = env.step(trajectory[i], pause=1) observation, reward, done, info = env.step(trajectory[i], pause=5)
print("Observation:", observation) print("Observation:", observation)
print("Reward:", reward) print("Reward:", reward)
print("Info:", info) print("Info:", info)