lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2023-12-21 10:42:18 +08:00
parent 70c04d3270 4a643abc31
commit de24fa91ba
6 changed files with 106 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@@ -160,4 +160,9 @@ instructor-embedding
# frontend cache
frontend/node_modules/
frontend/.next/
frontend/.idea
frontend/.idea
tags
snapshots
branch_flag
branch-config

101
desktop_env/controllers/setup.py
View File

@@ -1,6 +1,8 @@
import requests
import json
from typing import Dict, List
from typing import Any
class SetupController:
def __init__(self, http_server: str):
@@ -14,18 +16,42 @@ class SetupController:
...
}
"""
self._download_setup(config)
self._change_wallpaper(config)
for cfg in config:
config_type: str = cfg["type"]
parameters: Dict[str, Any] = cfg["parameters"]
# Assumes all the setup the functions should follow this name
# protocol
setup_function: str = "_{:}_setup".format(config_type)
assert hasattr(self, setup_function)
getattr(self, setup_function)(**parameters)
#self._download_setup(config)
#self._change_wallpaper(config)
# self._tidy_desktop(config) todo: implement this
self._open_setup(config)
#self._open_setup(config)
# can add other setup steps
def _download_setup(self, config):
if not config:
return
if not 'download' in config:
return
for url, path in config['download']:
def _download_setup(self, files: List[Dict[str, str]]):
"""
Args:
files (List[Dict[str, str]]): files to download. lisf of dict like
{
"url": str, the url to download
"path": str, the path on the VM to store the downloaded file
}
"""
#if not config:
#return
#if not 'download' in config:
#return
#for url, path in config['download']:
for f in files:
url: str = f["url"]
path: str = f["path"]
if not url or not path:
raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
@@ -44,12 +70,13 @@ class SetupController:
except requests.exceptions.RequestException as e:
print("An error occurred while trying to send the request:", e)
def _change_wallpaper(self, config):
if not config:
return
if not 'wallpaper' in config:
return
path = config['wallpaper']
def _change_wallpaper_setup(self, path: str):
#if not config:
#return
#if not 'wallpaper' in config:
#return
#path = config['wallpaper']
if not path:
raise Exception(f"Setup Wallpaper - Invalid path ({path}).")
@@ -68,29 +95,29 @@ class SetupController:
except requests.exceptions.RequestException as e:
print("An error occurred while trying to send the request:", e)
def _tidy_desktop(self, config):
def _tidy_desktop_setup(self, **config):
raise NotImplementedError
def _open_setup(self, config):
if not config:
return
if not 'open' in config:
return
for path in config['open']:
if not path:
raise Exception(f"Setup Open - Invalid path ({path}).")
def _open_setup(self, path: str):
#if not config:
#return
#if not 'open' in config:
#return
#for path in config['open']:
if not path:
raise Exception(f"Setup Open - Invalid path ({path}).")
payload = json.dumps({"path": path})
headers = {
'Content-Type': 'application/json'
}
payload = json.dumps({"path": path})
headers = {
'Content-Type': 'application/json'
}
# send request to server to open file
try:
response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
if response.status_code == 200:
print("Command executed successfully:", response.text)
else:
print("Failed to open file. Status code:", response.text)
except requests.exceptions.RequestException as e:
print("An error occurred while trying to send the request:", e)
# send request to server to open file
try:
response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
if response.status_code == 200:
print("Command executed successfully:", response.text)
else:
print("Failed to open file. Status code:", response.text)
except requests.exceptions.RequestException as e:
print("An error occurred while trying to send the request:", e)

2
desktop_env/envs/desktop_env.py
View File

@@ -60,7 +60,7 @@ class DesktopEnv(gym.Env):
def _start_emulator(self):
while True:
try:
output = subprocess.check_output(f"vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
output = subprocess.check_output("vmrun -T ws list", shell=True, stderr=subprocess.STDOUT)
output = output.decode()
if self.path_to_vm.lstrip("~/") in output:
print("VM is running.")

11
desktop_env/server/main.py
View File

@@ -4,9 +4,10 @@ import platform
import subprocess
import requests
import Xlib.display
#import Xlib.display
import pyautogui
from PIL import ImageGrab, Image
#from PIL import ImageGrab, Image
from PIL import Image
from flask import Flask, request, jsonify, send_file
app = Flask(__name__)
@@ -180,7 +181,11 @@ def open_file():
return f"File not found: {path}", 404
try:
os.startfile(path)
if platform.system()=="Windows":
os.startfile(path)
else:
open_cmd: str = "open" if platform.system()=="Darwin" else "xdg-open"
subprocess.Popen([open_cmd, str(path)])
return "File opened successfully"
except Exception as e:
return f"Failed to open {path}. Error: {e}", 500

30
evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
View File

@@ -3,17 +3,25 @@
"snapshot": "libreoffice_calc",
"instruction": "Fill the missing row and column which show the total value",
"source": "https://youtube.com/shorts/feldd-Pn48c?si=9xJiem2uAHm6Jshb",
"config": {
"download": [
[
"https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
"C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
]
},
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "C:\\Users\\tianbaox\\Desktop\\Quarterly_Product_Sales_by_Zone.xlsx"
}
}
],
"trajectory": "trajectories/f9584479-3d0d-4c79-affa-9ad7afdd8850",
"related_apps": [
"libreoffice calc"

19
main.py
View File

@@ -7,19 +7,16 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/37608790-6147-45d0-9f20-1137bb35703d.json", "r") as f:
with open("evaluation_examples/examples/f9584479-3d0d-4c79-affa-9ad7afdd8850.json", "r") as f:
example = json.load(f)
env = DesktopEnv(
# path_to_vm=r"""C:\Users\tianbaox\Downloads\Windows 10 x64\Windows 10 x64.vmx""",
path_to_vm=r"""C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx""",
# path_to_vm="/home/yuri/vmware/Ubuntu 64-bit/Ubuntu 64-bit.vmx",
action_space="computer_13",
snapshot_path="base_setup",
instruction=example["instruction"],
# config=example["config"],
# evaluator=example["evaluator"]
)
env = DesktopEnv( path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
, action_space="computer_13"
, snapshot_path="base_setup"
, instruction=example["instruction"]
, config=example["config"]
, evaluator=example["evaluator"]
)
# reset the environment to certain snapshot
observation = env.reset()