lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix OS examples annotated by Yitao

Browse Source
This commit is contained in:
Timothyxxx
2024-01-25 19:57:32 +08:00
parent 0c34fccc15
commit b9ae4174b1
24 changed files with 327 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/evaluators/getters/general.py
View File

@@ -9,8 +9,9 @@ def get_vm_command_line(env, config: Dict[str, str]):
vm_ip = env.vm_ip
port = 5000
command = config["command"]
shell = config.get("shell", False)
response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command})
response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command, "shell": shell})
if response.status_code == 200:
return response.json()["output"]

27
desktop_env/evaluators/metrics/__init__.py
View File

@@ -1,23 +1,28 @@
from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, check_font_size, \
check_enabled_experiments, check_history_deleted, is_expected_search_query
from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, \
check_font_size, \
check_enabled_experiments, check_history_deleted, is_expected_search_query
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
check_no_duplicates
from .general import exact_match, fuzzy_match
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json
from .general import exact_match, fuzzy_match, check_include_exclude
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
#from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file,check_qt_minimal_view, check_qt_max_volume, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \
check_qt_max_volume, \
check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, check_json_keybindings
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
check_json_keybindings
from .os import check_gnome_favorite_apps

18
desktop_env/evaluators/metrics/general.py
View File

@@ -1,8 +1,9 @@
import csv
import json
import functools
import json
import operator
import re
import sqlite3
from numbers import Number
from typing import Callable, Any, Union
from typing import Dict, List, Pattern
@@ -14,13 +15,14 @@ from rapidfuzz import fuzz
from .utils import _match_record, _match_value_to_rule
import sqlite3
def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
print(result, rules)
include = rules.get("include", [])
exclude = rules.get("exclude", [])
return all(r in result for r in include) and all(r not in result for r in exclude)
def exact_match(result, rules) -> float:
expect = rules["expected"]
print(result, expect)
@@ -36,6 +38,7 @@ def fuzzy_match(result, rules) -> float:
return fuzz.ratio(result, expect) / 100.
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
Args:
@@ -140,10 +143,10 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
return 0.
if "text" in rules:
match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"]\
else (lambda a, b: fuzz.ratio(a, b)/100.)
, rules["text"]
)
match_func: Callable[[str], Number] = functools.partial(operator.eq if rules["exact"] \
else (lambda a, b: fuzz.ratio(a, b) / 100.)
, rules["text"]
)
match_score: Number = 0
for elm in elements:
match_score = max(match_score, match_func(elm.text or None))
@@ -152,6 +155,7 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
return float(match_score)
# def check_existence(result: str, *args) -> float:
# return 1. - (result is None)
@@ -160,6 +164,7 @@ def run_sqlite3(result: str, rules: Dict[str, Any]) -> float:
cursor: sqlite3.Cursor = connection.execute(rules["sql"])
return float(cursor.fetchone()[0] or 0)
def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str]]]]) -> float:
"""
Args:
@@ -200,4 +205,3 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
value = value[k]
metric = metric and not _match_value_to_rule(value, r)
return metric

102
desktop_env/evaluators/metrics/os.py
View File

@@ -1,58 +1,20 @@
import os
import subprocess
def is_spotify_installed() -> bool:
# Use the 'which' command to check if the 'spotify' executable exists
try:
subprocess.check_output(['which', 'spotify'])
return True
except subprocess.CalledProcessError:
return False
def check_gnome_favorite_apps(apps_str: str, rule):
# parse the string like "['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']"
# to a list of strings
apps = eval(apps_str)
expected_apps = rule["expected"]
def is_dim_screen_inactive_off():
# Get the current value of "Dim screen when inactive" setting
result = subprocess.run(["gsettings", "get", "org.gnome.desktop.session", "idle-delay"], capture_output=True,
text=True)
if len(apps) != len(expected_apps):
return 0
# Check if the setting is set to "uint32 0"
if result.stdout.strip() == "uint32 0":
return 1 # Task successful
if set(apps) == set(expected_apps):
return 1
else:
return 0 # Task not successful
def evaluate_create_test_directory():
try:
# Specify the path to the directory
path = '/test'
# Check if the directory already exists
if os.path.exists(path):
return 0 # Directory already exists, task not successful
# Check if the user has sufficient permissions to create the directory
if not os.access("/", os.W_OK | os.X_OK):
return 0 # Insufficient permissions to create the directory, task not successful
return 1 # Task can be considered successful
except Exception as e:
print(f"Error: {str(e)}")
return 0 # Any other errors, task not successful
# TODO: create a file named test.txt and a directory dir1 at home dir before running this
def is_file_in_directory():
# Specify the paths of the file and directory
file_path = "/test.txt"
directory_path = "/dir1"
# Check if the file exists in the directory
if os.path.isfile(os.path.join(directory_path, os.path.basename(file_path))):
return 1 # Task successful
else:
return 0 # Task not successful
return 0
# TODO: log in to the system before running this
@@ -64,47 +26,6 @@ def is_logout_successful():
return 1 # Task successful
def is_do_not_disturb_mode_enabled():
try:
subprocess.run(["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"], check=True)
return 1 # Task successful
except subprocess.CalledProcessError:
return 0 # Task not successful
def get_default_browser(answer: str):
try:
result = subprocess.run(["xdg-settings", "get", "default-web-browser"], capture_output=True, text=True)
default_browser = result.stdout.strip()
return default_browser == answer
except subprocess.CalledProcessError:
return 0
# TODO: should confirm initial state, i.e. initial list of favorite apps & the first app
first_app = "thunderbird.desktop" # to be changed
def is_first_favorite_app_removed():
try:
result = subprocess.run(["gsettings", "get", "org.gnome.shell", "favorite-apps"], capture_output=True,
text=True)
output = result.stdout.strip()
# Remove brackets and spaces from the output
favorites = output[1:-1].replace(" ", "")
# Split output by comma to get individual favorite apps
favorite_apps = favorites.split(",")
if favorite_apps[0] != first_app:
return 1 # First favorite app removed
else:
return 0 # First favorite app not removed
except subprocess.CalledProcessError:
return 0 # Task not successful
def is_battery_percentage_displayed():
# GNOME schema and key for the setting
schema = "org.gnome.desktop.interface"
@@ -147,6 +68,3 @@ def check_auto_lock_settings():
except Exception as e:
return 0.