lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-05 15:55:41 +08:00
parent 3cbb57f24c eeb8a120d6
commit 26b7d9010d
19 changed files with 522 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,3 +1,3 @@
from .file import get_cloud_file, get_vm_file
from .file import get_cloud_file, get_vm_file, get_cache_file
from .misc import get_rule
from .vlc import get_vlc_playing_info

10
desktop_env/evaluators/getters/file.py
View File

@@ -41,3 +41,13 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
f.write(file)
return _path
def get_cache_file(env, config: Dict[str, str]) -> str:
"""
Config:
path (str): relative path in cache dir
"""
_path = os.path.join(env.cache_dir, config["path"])
assert os.path.exists(_path)
return _path

58
desktop_env/evaluators/getters/misc.py
View File

@@ -1,8 +1,66 @@
from typing import TypeVar
import platform
import subprocess
import ctypes
import os
import logging
logger = logging.getLogger("desktopenv.getters.misc")
R = TypeVar("Rule")
def get_rule(env, config: R) -> R:
"""
Returns the rule as-is.
"""
return config["rules"]
def get_desktop_path():
username = os.getlogin() # Get the current username
if platform.system() == "Windows":
return os.path.join("C:", "Users", username, "Desktop")
elif platform.system() == "Darwin": # macOS is identified as 'Darwin'
return os.path.join("/Users", username, "Desktop")
elif platform.system() == "Linux":
return os.path.join("/home", username, "Desktop")
else:
raise Exception("Unsupported operating system")
def get_wallpaper():
def get_wallpaper_windows():
SPI_GETDESKWALLPAPER = 0x73
MAX_PATH = 260
buffer = ctypes.create_unicode_buffer(MAX_PATH)
ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0)
return buffer.value
def get_wallpaper_macos():
script = """
tell application "System Events" to tell every desktop to get picture
"""
process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE)
output, error = process.communicate()
if error:
logger.error("Error: %s", error)
else:
return output.strip().decode('utf-8')
def get_wallpaper_linux():
try:
output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"])
return output.decode('utf-8').strip().replace('file://', '').replace("'", "")
except Exception as e:
logger.error("Error: %s", e)
return None
os_name = platform.system()
if os_name == 'Windows':
return get_wallpaper_windows()
elif os_name == 'Darwin':
return get_wallpaper_macos()
elif os_name == 'Linux':
return get_wallpaper_linux()
else:
return "Unsupported OS"

1
desktop_env/evaluators/metrics/__init__.py
View File

@@ -6,3 +6,4 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
from .pdf import check_pdf_pages
from .libreoffice import check_libre_locale
from .vlc import is_vlc_playing
from .general import check_csv

11
desktop_env/evaluators/metrics/chrome.py
View File

@@ -5,6 +5,9 @@ import sqlite3
from playwright.sync_api import sync_playwright
import logging
logger = logging.getLogger("desktopenv.metrics.chrome")
"""
WARNING:
1. Functions from this script assume that no account is registered on Chrome, otherwise the default file path needs to be changed.
@@ -36,7 +39,7 @@ def get_default_search_engine():
'Google')
return search_engine
except Exception as e:
print(f"Error: {e}")
logger.error(f"Error: {e}")
return "Google"
@@ -61,7 +64,7 @@ def get_cookie_data():
return cookies
except Exception as e:
print(f"Error: {e}")
logger.error(f"Error: {e}")
return None
@@ -85,7 +88,7 @@ def get_bookmarks():
return bookmarks
except Exception as e:
print(f"Error: {e}")
logger.error(f"Error: {e}")
return None
@@ -117,7 +120,7 @@ def get_extensions_installed_from_shop():
manifest = json.load(file)
manifests.append(manifest)
except json.JSONDecodeError:
print(f"Error reading {manifest_path}")
logger.error(f"Error reading {manifest_path}")
return manifests

6
desktop_env/evaluators/metrics/docs.py
View File

@@ -4,6 +4,8 @@ from typing import List, Dict, Any
from docx import Document
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
import logging
logger = logging.getLogger("desktopenv.metric.docs")
def find_default_font(config_file_path, rules):
"""Find the default font in LibreOffice Writer."""
@@ -23,7 +25,7 @@ def find_default_font(config_file_path, rules):
for value in prop.findall('value', namespace):
default_font = value.text
except Exception as e:
print(f"Error: {e}")
logger.error(f"Error: {e}")
return 1 if default_font == expected_font else 0
@@ -192,4 +194,4 @@ def compare_contains_image(docx_file1, docx_file2):
# print(result)
# config_path = "/home/[username]/.config/libreoffice/4/user/registrymodifications.xcu"
# print(find_default_font("Ani", config_path))
# print(find_default_font("Ani", config_path))

79
desktop_env/evaluators/metrics/general.py
View File

@@ -1,57 +1,30 @@
import platform
import subprocess
import ctypes
import os
import csv
from typing import Dict, List
def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
return all(k in item and item[k]==val for k, val in pattern.items())
# todo: move to getter module
def get_desktop_path():
username = os.getlogin() # Get the current username
if platform.system() == "Windows":
return os.path.join("C:", "Users", username, "Desktop")
elif platform.system() == "Darwin": # macOS is identified as 'Darwin'
return os.path.join("/Users", username, "Desktop")
elif platform.system() == "Linux":
return os.path.join("/home", username, "Desktop")
else:
raise Exception("Unsupported operating system")
def get_wallpaper():
def get_wallpaper_windows():
SPI_GETDESKWALLPAPER = 0x73
MAX_PATH = 260
buffer = ctypes.create_unicode_buffer(MAX_PATH)
ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0)
return buffer.value
def get_wallpaper_macos():
script = """
tell application "System Events" to tell every desktop to get picture
"""
process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE)
output, error = process.communicate()
if error:
print("Error:", error)
else:
return output.strip().decode('utf-8')
def get_wallpaper_linux():
try:
output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"])
return output.decode('utf-8').strip().replace('file://', '').replace("'", "")
except Exception as e:
print("Error:", e)
return None
os_name = platform.system()
if os_name == 'Windows':
return get_wallpaper_windows()
elif os_name == 'Darwin':
return get_wallpaper_macos()
elif os_name == 'Linux':
return get_wallpaper_linux()
else:
return "Unsupported OS"
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
Args:
result (str): path to csv file
rules (Dict[str, List[Dict[str, str]]]): dict like
{
"expect": [{key: value}]
"unexpect": [{key: value}]
}
Returns:
float
"""
expect_metrics = [False] * len(rules.get("expect", []))
unexpect_metric = True
with open(result) as f:
reader = csv.DictReader(f)
for rcd in reader:
for i, r in enumerate(rules.get("expect", [])):
expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd)
unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", []))
return float(all(expect_metrics) and unexpect_metric)

14
desktop_env/evaluators/metrics/table.py
View File

@@ -10,6 +10,8 @@ from typing import Dict, List
from typing import Any, Union
from numbers import Number
import logging
logger = logging.getLogger("desktopenv.metric.table")
def compare_table(actual: str, expected: str, **options) -> float:
"""
@@ -32,7 +34,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
print("Normal Contents Metric: {:}".format(metric))
logger.debug("Normal Contents Metric: {:}".format(metric))
features: List[str] = options.get("features", [])
for ftr in features:
@@ -43,12 +45,12 @@ def compare_table(actual: str, expected: str, **options) -> float:
sp1 = load_sparklines(actual)
sp2 = load_sparklines(expected)
new_metric: bool = sp1 == sp2
print("Sparkline Metric: {:}".format(new_metric))
logger.debug("Sparkline Metric: {:}".format(new_metric))
elif ftr=="chart":
charts1 = load_charts(workbook1, **options)
charts2 = load_charts(workbook2, **options)
new_metric: bool = charts1 == charts2
print("Chart Metric: {:}".format(new_metric))
logger.debug("Chart Metric: {:}".format(new_metric))
elif ftr=="number_format":
number_formats1: List[str] = [ c.number_format.lower()\
for col in workbook1.active.iter_cols()\
@@ -61,7 +63,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
if c.data_type=="n"
]
new_metric: bool = number_formats1==number_formats2
print("Number Format Metric: {:}".format(new_metric))
logger.debug("Number Format Metric: {:}".format(new_metric))
else:
raise NotImplementedError("Unsupported xlsx feature: {:}".format(ftr))
metric = metric and new_metric
@@ -79,7 +81,7 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
expected_name: str = worksheet_names[r["sheet_idx"]]
actual_name: str = r["sheet_name"]
metric: bool = expected_name == actual_name
print("Assertion: {:d}.{:} is {:} - {:}".format(r["sheet_idx"], actual_name, expected_name, metric))
logger.debug("Assertion: {:d}.{:} is {:} - {:}".format(r["sheet_idx"], actual_name, expected_name, metric))
passes = passes and metric
elif r["type"] == "sheet_data":
if isinstance(r["sheet_idx0"], int):
@@ -99,7 +101,7 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
sheet_idx: int = int(sheet_idx)
df2: pd.DataFrame = pd.read_excel(file_name, sheet_idx)
metric: bool = df1.equals(df2)
print("Assertion: {:} == {:} - {:}".format(r["sheet_idx0"], r["sheet_idx1"], metric))
logger.debug("Assertion: {:} == {:} - {:}".format(r["sheet_idx0"], r["sheet_idx1"], metric))
passes = passes and metric
else:
raise NotImplementedError("Unimplemented sheet check: {:}".format(r["type"]))

3
desktop_env/evaluators/metrics/utils.py
View File

@@ -11,6 +11,9 @@ from openpyxl.chart._chart import ChartBase
from typing import Dict, List, Set
from typing import Any
import logging
logger = logging.getLogger("desktopenv.metrics.utils")
_xlsx_namespaces = [ ("x14", "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main")
, ("xm", "http://schemas.microsoft.com/office/excel/2006/main")
]

10
desktop_env/evaluators/metrics/vlc.py
View File

@@ -5,6 +5,8 @@ import pygetwindow as gw
import pyautogui
from typing import Dict
import logging
logger = logging.getLogger("desktopenv.metrics.vlc")
def get_vlc_config(setting_name):
"""
@@ -25,7 +27,7 @@ def get_vlc_config(setting_name):
config_path = paths.get(os_type)
if not config_path or not os.path.exists(config_path):
print("VLC config file not found for this operating system.")
logger.warning("VLC config file not found for this operating system.")
return None
try:
@@ -34,7 +36,7 @@ def get_vlc_config(setting_name):
if line.startswith(setting_name):
return line.strip()
except IOError as e:
print(f"Error reading config file: {e}")
logger.error(f"Error reading config file: {e}")
return None
@@ -77,8 +79,8 @@ def is_vlc_fullscreen():
except IndexError:
# VLC window not found
print("VLC window not found.")
logger.error("VLC window not found.")
return False
except Exception as e:
print(f"An error occurred: {e}")
logger.error(f"An error occurred: {e}")
return False