lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2023-12-25 20:29:20 +08:00
parent ba77c276e6 e891eedfde
commit ade9002da4
80 changed files with 835 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
desktop_env/__init__.py
View File

@@ -0,0 +1 @@

37
desktop_env/controllers/setup.py
View File

@@ -8,6 +8,7 @@ import os.path
from typing import Dict, List
from typing import Any
class SetupController:
def __init__( self
, http_server: str
@@ -41,10 +42,10 @@ class SetupController:
assert hasattr(self, setup_function)
getattr(self, setup_function)(**parameters)
#self._download_setup(config)
#self._change_wallpaper(config)
# self._download_setup(config)
# self._change_wallpaper(config)
# self._tidy_desktop(config) todo: implement this
#self._open_setup(config)
# self._open_setup(config)
# can add other setup steps
def _download_setup(self, files: List[Dict[str, str]]):
@@ -57,11 +58,11 @@ class SetupController:
}
"""
#if not config:
#return
#if not 'download' in config:
#return
#for url, path in config['download']:
# if not config:
# return
# if not 'download' in config:
# return
# for url, path in config['download']:
for f in files:
url: str = f["url"]
path: str = f["path"]
@@ -119,12 +120,12 @@ class SetupController:
print("An error occurred while trying to send the request:", e)
def _change_wallpaper_setup(self, path: str):
#if not config:
#return
#if not 'wallpaper' in config:
#return
# if not config:
# return
# if not 'wallpaper' in config:
# return
#path = config['wallpaper']
# path = config['wallpaper']
if not path:
raise Exception(f"Setup Wallpaper - Invalid path ({path}).")
@@ -147,11 +148,11 @@ class SetupController:
raise NotImplementedError
def _open_setup(self, path: str):
#if not config:
#return
#if not 'open' in config:
#return
#for path in config['open']:
# if not config:
# return
# if not 'open' in config:
# return
# for path in config['open']:
if not path:
raise Exception(f"Setup Open - Invalid path ({path}).")

40
desktop_env/envs/desktop_env.py
View File

@@ -3,23 +3,24 @@ from __future__ import annotations
import os
import subprocess
import time
#import uuid
#import platform
# import uuid
# import platform
from typing import List, Dict
from typing import Callable, Any, Optional
import tempfile
import gymnasium as gym
#import requests
# import requests
from desktop_env.controllers.python import PythonController
from desktop_env.controllers.setup import SetupController
#from desktop_env.evaluators import eval_funcs
# from desktop_env.evaluators import eval_funcs
from desktop_env.evaluators import metrics, getters
Metric = Callable[[Any, Any], float]
Getter = Callable[[gym.Env, Dict[str, Any]], Any]
def _execute_command(command: List[str]) -> None:
if command[:4] == ["vmrun", "-T", "ws", "start"]:
p = subprocess.Popen(command)
@@ -104,7 +105,7 @@ class DesktopEnv(gym.Env):
output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
output = output.decode()
output: List[str] = output.splitlines()
#if self.path_to_vm.lstrip("~/") in output:
# if self.path_to_vm.lstrip("~/") in output:
if self.path_to_vm in output:
print("VM is running.")
break
@@ -132,9 +133,9 @@ class DesktopEnv(gym.Env):
_execute_command(["vmrun", "-T", "ws" "snapshot", self.path_to_vm, self.snapshot_path])
def _get_screenshot(self):
#random_uuid = str(uuid.uuid4())
#os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True)
#image_path = os.path.join("tmp", random_uuid, "screenshot.png")
# random_uuid = str(uuid.uuid4())
# os.makedirs(os.path.join("tmp", random_uuid), exist_ok=True)
# image_path = os.path.join("tmp", random_uuid, "screenshot.png")
image_path: str = os.path.join(self.tmp_dir, "screenshots", "{:d}.png".format(self._step_no))
# Get the screenshot and save to the image_path
@@ -173,9 +174,10 @@ class DesktopEnv(gym.Env):
self.action_history.clear()
print("Setup new temp dir...")
self.tmp_dir = tempfile.mkdtemp( prefix="{:d}.{:}.".format(self._traj_no, self.task_id)
, dir=self.tmp_dir_base
)
self.tmp_dir = tempfile.mkdtemp(
prefix="{:d}.{:}.".format(self._traj_no, self.task_id),
dir=self.tmp_dir_base
)
os.makedirs(os.path.join(self.tmp_dir, "screenshots"))
print("Reverting to snapshot to {}...".format(self.snapshot_path))
@@ -224,14 +226,14 @@ class DesktopEnv(gym.Env):
"""
# todo: make this more flexible by refactoring
#eval_func = eval_funcs[self.evaluator["func"]]
#eval_func_vars = {}
#
#for var_name, file_info in self.evaluator["paths"].items():
#path = copy_file_to_local(file_info)
#eval_func_vars[var_name] = path
#
#return eval_func(**eval_func_vars)
# eval_func = eval_funcs[self.evaluator["func"]]
# eval_func_vars = {}
#
# for var_name, file_info in self.evaluator["paths"].items():
# path = copy_file_to_local(file_info)
# eval_func_vars[var_name] = path
#
# return eval_func(**eval_func_vars)
result = self.result_getter(self, self.evaluator["result"])
expected = self.expected_getter(self, self.evaluator["expected"])

9
desktop_env/server/main.py
View File

@@ -4,9 +4,9 @@ import platform
import subprocess
import requests
#import Xlib.display
# import Xlib.display
import pyautogui
#from PIL import ImageGrab, Image
# from PIL import ImageGrab, Image
from PIL import Image
from flask import Flask, request, jsonify, send_file
@@ -47,7 +47,6 @@ def capture_screen_with_cursor():
# Ensure the screenshots directory exists
os.makedirs(os.path.dirname(file_path), exist_ok=True)
# fixme: This is a temporary fix for the cursor not being captured on Windows and Linux
if user_platform == "Windows" or user_platform == "Linux":
def _download_image(url, path):
@@ -191,10 +190,10 @@ def open_file():
return f"File not found: {path}", 404
try:
if platform.system()=="Windows":
if platform.system() == "Windows":
os.startfile(path)
else:
open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open"
open_cmd: str = "open" if platform.system() == "Darwin" else "xdg-open"
subprocess.Popen([open_cmd, str(path)])
return "File opened successfully"
except Exception as e: