lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2024-01-05 16:14:38 +08:00
parent eeb8a120d6 fbb4918734
commit 69b29e4e55
18 changed files with 491 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
desktop_env/controllers/python.py
View File

@@ -82,7 +82,8 @@ class PythonController:
y = parameters["y"]
if "num_clicks" in parameters:
num_clicks = parameters["num_clicks"]
self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})")
self.execute_python_command(
f"pyautogui.click(button='{button}', x={x}, y={y}, clicks={num_clicks})")
else:
self.execute_python_command(f"pyautogui.click(button='{button}', x={x}, y={y})")
elif "button" in parameters and "x" not in parameters and "y" not in parameters:
@@ -145,7 +146,8 @@ class PythonController:
if "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
self.execute_python_command(f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)")
self.execute_python_command(
f"pyautogui.dragTo({x}, {y}, duration=1.0, button='left', mouseDownUp=True)")
elif action_type == "SCROLL":
# todo: check if it is related to the operating system, as https://github.com/TheDuckAI/DuckTrack/blob/main/ducktrack/playback.py pointed out
@@ -208,3 +210,16 @@ class PythonController:
else:
raise Exception(f"Unknown action type: {action_type}")
def get_vlc_status(self, host='localhost', port=8080, password='password'):
url = f'http://{host}:{port}/requests/status.xml'
response = requests.get(url, auth=('', password))
if response.status_code == 200:
print("File downloaded successfully")
return response.content
else:
print("Failed to get vlc status. Status code:", response.status_code)
return None

73
desktop_env/controllers/setup.py
View File

@@ -14,12 +14,11 @@ logger = logging.getLogger("desktopenv.setup")
import traceback
class SetupController:
def __init__( self
, http_server: str
, cache_dir: str
):
self.http_server = http_server + "/setup"
def __init__(self, http_server: str, cache_dir: str):
self.http_server: str = http_server
self.http_server_setup_root = http_server + "/setup"
self.cache_dir: str = cache_dir
def reset_cache_dir(self, cache_dir: str):
self.cache_dir = cache_dir
@@ -52,6 +51,33 @@ class SetupController:
# self._open_setup(config)
# can add other setup steps
# ZDY_COMMENT: merged with launch
#def _command_setup(self, command: str):
#"""
#Directly send a command into the virtual machine os for setting up.
#"""
#payload = json.dumps({"command": command})
#headers = {
#'Content-Type': 'application/json'
#}
#timeout = 5
#timout_whitelist = ["vlc"]
#
#try:
#
#response = requests.post(self.http_server + "/execute", headers=headers, data=payload, timeout=timeout)
#if response.status_code == 200:
#print("Command executed successfully:", response.text)
#else:
#print("Failed to execute command. Status code:", response.status_code)
#except requests.exceptions.Timeout as e:
#if command in timout_whitelist:
#print("Command executed successfully:", command)
#else:
#print("An error occurred while trying to execute the command:", e)
#except requests.exceptions.RequestException as e:
#print("An error occurred while trying to execute the command:", e)
def _download_setup(self, files: List[Dict[str, str]]):
"""
Args:
@@ -70,12 +96,9 @@ class SetupController:
for f in files:
url: str = f["url"]
path: str = f["path"]
cache_path: str = os.path.join( self.cache_dir
, "{:}_{:}".format(
uuid.uuid5(uuid.NAMESPACE_URL, url)
, os.path.basename(path)
)
)
cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format(
uuid.uuid5(uuid.NAMESPACE_URL, url),
os.path.basename(path)))
if not url or not path:
raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
@@ -101,22 +124,22 @@ class SetupController:
if not downloaded:
raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
#payload = json.dumps({"url": url, "path": path})
#headers = {
#'Content-Type': 'application/json'
#}
# payload = json.dumps({"url": url, "path": path})
# headers = {
# 'Content-Type': 'application/json'
# }
form = MultipartEncoder( { "file_path": path
, "file_data": (os.path.basename(path), open(cache_path, "rb"))
}
)
form = MultipartEncoder({
"file_path": path,
"file_data": (os.path.basename(path), open(cache_path, "rb"))
})
headers = {"Content-Type": form.content_type}
logger.debug(form.content_type)
# send request to server to upload file
try:
logger.debug("REQUEST ADDRESS: %s", self.http_server + "/upload")
response = requests.post(self.http_server + "/upload", headers=headers, data=form)
logger.debug("REQUEST ADDRESS: %s", self.http_server_setup_root + "/upload")
response = requests.post(self.http_server_setup_root + "/upload", headers=headers, data=form)
if response.status_code == 200:
logger.info("Command executed successfully: %s", response.text)
else:
@@ -141,7 +164,7 @@ class SetupController:
# send request to server to change wallpaper
try:
response = requests.post(self.http_server + "/change_wallpaper", headers=headers, data=payload)
response = requests.post(self.http_server_setup_root + "/change_wallpaper", headers=headers, data=payload)
if response.status_code == 200:
logger.info("Command executed successfully: %s", response.text)
else:
@@ -168,7 +191,7 @@ class SetupController:
# send request to server to open file
try:
response = requests.post(self.http_server + "/open_file", headers=headers, data=payload)
response = requests.post(self.http_server_setup_root + "/open_file", headers=headers, data=payload)
if response.status_code == 200:
logger.info("Command executed successfully: %s", response.text)
else:
@@ -184,7 +207,7 @@ class SetupController:
headers = {"Content-Type": "application/json"}
try:
response = requests.post(self.http_server + "/launch", headers=headers, data=payload)
response = requests.post(self.http_server_setup_root + "/launch", headers=headers, data=payload)
if response.status_code == 200:
logger.info("Command executed successfully: %s", response.text)
else:
@@ -200,7 +223,7 @@ class SetupController:
headers = {"Content-Type": "application/json"}
try:
response = requests.post(self.http_server + "/execute", headers=headers, data=payload)
response = requests.post(self.http_server_setup_root + "/execute", headers=headers, data=payload)
if response.status_code == 200:
results: Dict[str, str] = response.json()
if stdout:

3
desktop_env/envs/desktop_env.py
View File

@@ -76,7 +76,8 @@ class DesktopEnv(gym.Env):
# Initialize emulator and controller
logger.info("Initializing...")
self._start_emulator()
self.host = f"http://{self._get_vm_ip()}:5000"
self.vm_ip = self._get_vm_ip()
self.host = f"http://{self.vm_ip}:5000"
self.controller = PythonController(http_server=self.host)
self.setup_controller = SetupController(http_server=self.host, cache_dir=self.cache_dir)

1
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,2 +1,3 @@
from .file import get_cloud_file, get_vm_file, get_cache_file
from .misc import get_rule
from .vlc import get_vlc_playing_info

2
desktop_env/evaluators/getters/file.py
View File

@@ -3,6 +3,7 @@ from typing import Dict
import os
import requests
def get_cloud_file(env, config: Dict[str, str]) -> str:
"""
Config:
@@ -25,6 +26,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str:
return _path
def get_vm_file(env, config: Dict[str, str]) -> str:
"""
Config:

20
desktop_env/evaluators/getters/vlc.py Normal file
View File

@@ -0,0 +1,20 @@
import os
from typing import Dict
def get_vlc_playing_info(env, config: Dict[str, str]):
"""
Gets the current playing information from VLC's HTTP interface.
"""
_path = os.path.join(env.cache_dir, config["dest"])
host = env.vm_ip
port = 8080
password = 'password'
content = env.controller.get_vlc_status(host, port, password)
print("content: ", content)
with open(_path, "wb") as f:
f.write(content)
return _path

1
desktop_env/evaluators/metrics/__init__.py
View File

@@ -5,4 +5,5 @@ from .docs import compare_font_names, compare_subscript_contains, has_page_numbe
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .pdf import check_pdf_pages
from .libreoffice import check_libre_locale
#from .vlc import is_vlc_playing
from .general import check_csv

22
desktop_env/evaluators/metrics/gimp.py Normal file
View File

@@ -0,0 +1,22 @@
import os
def get_gimp_export_path():
# Path to GIMP's configuration file. This example assumes GIMP version 2.10.
# You need to adjust the path according to the GIMP version and user's file system.
gimp_config_file = os.path.expanduser("~/.config/GIMP/2.10/gimprc")
try:
# Open and read the configuration file
with open(gimp_config_file, 'r') as file:
for line in file:
# Search for the default export path setting
if "default-export-path" in line:
# Extract the current path from the line (assuming it's enclosed in quotes)
current_path = line.split('"')[1]
# Compare the current path with the expected path
return current_path
except FileNotFoundError:
# Handle the case where the configuration file is not found
print("GIMP configuration file not found")
return False

34
desktop_env/evaluators/metrics/vlc.py
View File

@@ -1,14 +1,14 @@
import os
import platform
import requests
from xml.etree import ElementTree
import pygetwindow as gw
import pyautogui
from typing import Dict
import logging
logger = logging.getLogger("desktopenv.metrics.vlc")
def read_vlc_config(setting_name):
def get_vlc_config(setting_name):
"""
Reads the VLC configuration file to check for a specific setting.
@@ -41,24 +41,22 @@ def read_vlc_config(setting_name):
return None
def get_vlc_playing_info(host='localhost', port=8080, password='password'):
def is_vlc_playing(actual: str, rule: Dict[str, str]) -> float:
"""
Gets the current playing information from VLC's HTTP interface.
Checks if VLC is currently playing a file.
"""
url = f'http://{host}:{port}/requests/status.xml'
try:
response = requests.get(url, auth=('', password))
if response.status_code == 200:
tree = ElementTree.fromstring(response.content)
status = tree.find('state').text
if status == 'playing':
file_info = tree.find('information/category[@name="meta"]/info[@name="filename"]').text
return status, file_info
return status, None
except Exception as e:
logger.error(f"Error: {e}")
with open(actual, 'rb') as file:
actual_status = file.read().decode('utf-8')
return None, None
tree = ElementTree.fromstring(actual_status)
status = tree.find('state').text
if status == 'playing':
file_info = tree.find('information/category[@name="meta"]/info[@name="filename"]').text
print("file_info: ", file_info)
if file_info:
return 1 if file_info.endswith(rule['expected']) else 0
else:
return 0
def is_vlc_fullscreen():
@@ -86,5 +84,3 @@ def is_vlc_fullscreen():
except Exception as e:
logger.error(f"An error occurred: {e}")
return False