lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update todos

Browse Source
This commit is contained in:
Timothyxxx
2024-03-14 22:36:33 +08:00
parent 44ff027801
commit 35ed7cec89
2 changed files with 14 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
desktop_env/envs/desktop_env.py
View File

@@ -53,7 +53,7 @@ class DesktopEnv(gym.Env):
def __init__( def __init__(
self, self,
path_to_vm: str, path_to_vm: str,
snapshot_name: str ="init_state", snapshot_name: str = "init_state",
action_space: str = "computer_13", action_space: str = "computer_13",
tmp_dir: str = "tmp", tmp_dir: str = "tmp",
cache_dir: str = "cache", cache_dir: str = "cache",
@@ -75,7 +75,7 @@ class DesktopEnv(gym.Env):
self.snapshot_name = snapshot_name self.snapshot_name = snapshot_name
self.tmp_dir_base: str = tmp_dir self.tmp_dir_base: str = tmp_dir
self.cache_dir_base: str = cache_dir self.cache_dir_base: str = cache_dir
self.vm_screen_size = screen_size self.vm_screen_size = screen_size # todo: add the logic to get the screen size from the VM
self.headless = headless self.headless = headless
os.makedirs(self.tmp_dir_base, exist_ok=True) os.makedirs(self.tmp_dir_base, exist_ok=True)

34
main.py
View File

@@ -4,7 +4,7 @@ import logging
import os import os
import sys import sys
import time import time
import argparse
from desktop_env.envs.desktop_env import DesktopEnv from desktop_env.envs.desktop_env import DesktopEnv
# Logger Configs {{{ # # Logger Configs {{{ #
@@ -46,39 +46,29 @@ def human_agent():
""" """
Runs the Gym environment with human input. Runs the Gym environment with human input.
""" """
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--path', type=str, required=True, help="Path to the virtual machine .vmx file.")
parser.add_argument('-s', '--snapshot', type=str, help="Name of the snapshot to restore.")
parser.add_argument('-e', '--example', type=str, help="Path to the example json file.")
args = parser.parse_args(sys.argv[1:])
example_path = args.example if args.example is not None and os.path.exists(args.example) else \ with open("evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json", "r", encoding="utf-8") as f:
'evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json'
with open(example_path, "r") as f:
example = json.load(f) example = json.load(f)
# change to your customized snapshot example["snapshot"] = "exp_v5"
if args.snapshot is not None: example["snapshot"] = args.snapshot
assert os.path.exists(args.path), "The specified path to the .vmx file does not exist."
env = DesktopEnv( env = DesktopEnv(
path_to_vm=args.path, path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu3\Ubuntu3.vmx",
action_space="computer_13", action_space="computer_13",
task_config=example task_config=example
) )
# reset the environment to certain snapshot # reset the environment to certain snapshot
observation = env.reset() observation = env.reset()
logger.info('\x1b[32m[TASK INSTRUCTION]: \x1b[32;3m%s\x1b[0m', example["instruction"])
done = False done = False
trajectory = [ trajectory = [
# { {
# "action_type": "MOVE_TO", "action_type": "MOVE_TO", #
# "parameters": { "parameters": {
# "x": 754, "x": 754,
# "y": 1057 "y": 1057
# } }
# }, },
# {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}} {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
] ]
for i in range(len(trajectory)): for i in range(len(trajectory)):