lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add impress examples, format the import

Browse Source
This commit is contained in:
Timothyxxx
2024-01-30 03:25:27 +08:00
parent 20f48759fd
commit cb7643713e
6 changed files with 257 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,13 +1,25 @@
from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \
get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \
get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info
from .chrome import (
get_default_search_engine,
get_cookie_data,
get_bookmarks,
get_open_tabs_info,
get_pdf_from_url,
get_shortcuts_on_desktop,
get_history,
get_enabled_experiments,
get_chrome_language,
get_chrome_font_size,
get_profile_name,
get_number_of_search_results,
get_googledrive_file,
get_active_tab_info
)
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line, get_vm_terminal_output
from .gimp import get_gimp_config_file
from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config
from .vscode import get_vscode_config
from .impress import get_audio_in_slide
from .gimp import get_gimp_config_file

132
desktop_env/evaluators/metrics/__init__.py
View File

@@ -1,26 +1,58 @@
from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, \
check_font_size, \
check_enabled_experiments, check_history_deleted, is_expected_search_query
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines
from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
check_no_duplicates, compare_init_lines
from .general import exact_match, fuzzy_match
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json, check_list
from .general import exact_match, fuzzy_match, check_include_exclude
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .gimp import compare_images
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel, check_transition
from .table import compare_table, compare_csv
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .basic_os import (
check_gnome_favorite_apps,
is_utc_0,
check_text_enlarged,
check_moved_jpgs
)
from .chrome import (
is_expected_tabs,
is_expected_bookmarks,
compare_pdfs,
is_cookie_deleted,
is_shortcut_on_desktop,
check_font_size,
check_enabled_experiments,
check_history_deleted,
is_expected_search_query
)
from .docs import (
compare_font_names,
compare_subscript_contains,
has_page_numbers_in_footers,
compare_docx_lines,
evaluate_colored_words_in_tables,
check_highlighted_words,
evaluate_strike_through_last_paragraph,
evaluate_conversion,
evaluate_spacing,
check_italic_font_size_14,
evaluate_alignment,
get_unique_train_ids,
check_no_duplicates,
compare_init_lines
)
from .docs import (
find_default_font,
contains_page_break,
compare_docx_files,
compare_docx_tables,
compare_line_spacing,
compare_insert_equation,
compare_highlighted_text,
is_first_line_centered,
check_file_exists,
compare_contains_image
)
from .general import (
check_csv,
check_accessibility_tree,
run_sqlite3,
check_json,
check_list,
exact_match,
fuzzy_match,
check_include_exclude
)
from .gimp import (
check_brightness_decrease_and_structure_sim,
check_contrast_increase_and_structure_sim,
@@ -34,11 +66,53 @@ from .gimp import (
check_triangle_position,
check_structure_sim,
check_config_status,
compare_images,
increase_saturation,
decrease_brightness,
check_file_exists,
compare_triangle_positions
)
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \
check_qt_max_volume, \
check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .slides import (
check_presenter_console_disable,
check_image_stretch_and_center,
check_slide_numbers_color,
compare_pptx_files,
check_strikethrough,
check_slide_orientation_Portrait,
evaluate_presentation_fill_to_rgb_distance,
check_left_panel,
check_transition,
check_page_number_colors
)
from .table import (
compare_table,
compare_csv
)
from .thunderbird import (
check_thunderbird_prefs,
check_thunderbird_filter
)
from .vlc import (
is_vlc_playing,
is_vlc_recordings_folder,
is_vlc_fullscreen,
compare_images,
compare_audios,
compare_videos,
check_qt_bgcone,
check_one_instance_when_started_from_file,
check_qt_minimal_view,
check_qt_max_volume,
check_qt_slider_colours,
check_global_key_play_pause
)
from .vscode import (
compare_text_file,
compare_config,
compare_answer,
is_extension_installed,
check_json_settings,
check_json_keybindings
from .basic_os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs
)

2
desktop_env/evaluators/metrics/docs.py
View File

@@ -2,6 +2,7 @@ import logging
import os
import re
import xml.etree.ElementTree as ET
import zipfile
from typing import List, Dict, Any
from docx import Document
@@ -10,7 +11,6 @@ from docx.shared import RGBColor
from odf.opendocument import load
from odf.text import P
from odf.text import Span
import zipfile
logger = logging.getLogger("desktopenv.metric.docs")

41
desktop_env/evaluators/metrics/gimp.py
View File

@@ -211,7 +211,10 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
img_tgt_normalized = normalize_brightness(img_tgt, target_brightness)
structure_same = structure_check_by_mse(img_src_normalized, img_tgt_normalized)
return brightness_reduced and structure_same
if brightness_reduced and structure_same:
return 1.
else:
return 0.
def check_saturation_increase_and_structure_sim(src_path, tgt_path):
@@ -240,7 +243,10 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path):
else:
structure_same = False
return saturation_increased and structure_same
if saturation_increased and structure_same:
return 1.
else:
return 0.
def check_file_exists_and_structure_sim(src_path, tgt_path):
@@ -295,7 +301,10 @@ def check_triangle_position(tgt_path):
tolerance = 0.05 * np.array(img_array.shape[:2])
middle = np.all(np.abs(centroid - image_center) < tolerance)
return middle
if bool(middle):
return 1.
else:
return 0.
def check_structure_sim(src_path, tgt_path):
@@ -326,7 +335,10 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path):
# Check structure
structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.65)
return higher_contrast and structure_same
if higher_contrast and structure_same:
return 1.
else:
return 0.
def check_config_status(actual_config_path, rule):
@@ -342,13 +354,13 @@ def check_config_status(actual_config_path, rule):
items = line.strip().lstrip('(').rstrip(')\n').split()
if isinstance(rule["key"], str):
if items[0] == rule["key"] and items[-1] == rule["value"]:
return True
return 1.
elif isinstance(rule["key"], list) and len(rule["key"]) == 2:
if items[0] == rule["key"][0] \
and items[1] == rule["key"][1] \
and items[-1] == rule["value"]:
return True
return False
return 1.
return 0.
def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None):
@@ -374,7 +386,10 @@ def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=Non
resized_target_image = target_image.resize(source_image.size)
structure_same = structure_check_by_ssim(source_image, resized_target_image)
return width_same and height_same and structure_same
if width_same and height_same and structure_same:
return 1.
else:
return 0.
def check_palette_and_structure_sim(src_path, tgt_path):
@@ -390,7 +405,10 @@ def check_palette_and_structure_sim(src_path, tgt_path):
target_image = Image.open(tgt_path)
source_image = source_image.convert('RGB')
structure_same = structure_check_by_ssim(source_image, target_image)
return palette_based and structure_same
if palette_based and structure_same:
return 1.
else:
return 0.
def check_textbox_on_leftside(src_path):
@@ -412,7 +430,10 @@ def check_textbox_on_leftside(src_path):
break # Stop after finding the first dark pixel in this row
# Here we define "almost" on the left side as being within the left 5% of the image
return left_most_dark_pixel < width * 0.05
if left_most_dark_pixel < width * 0.05:
return 1.
else:
return 0.
def check_image_mirror(src_path, tgt_path):

47
desktop_env/evaluators/metrics/slides.py
View File

@@ -318,3 +318,50 @@ def check_transition(pptx_file, rules):
return 0.
else:
return 0.
def check_page_number_colors(pptx_file, rules):
color = rules["color"]
def is_red(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return r > g + threshold and r > b + threshold
def is_blue(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return b > g + threshold and b > r + threshold
def is_green(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return g > r + threshold and g > b + threshold
def is_black(rgb_str, threshold=50):
r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16)
return r < threshold and g < threshold and b < threshold
with zipfile.ZipFile(pptx_file, 'r') as zip_ref:
slide_master_name = 'ppt/slideMasters/slideMaster1.xml'
with zip_ref.open(slide_master_name) as slide_master_file:
tree = ET.parse(slide_master_file)
root = tree.getroot()
namespaces = {
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
}
color_elems = root.findall('.//a:solidFill//a:srgbClr', namespaces)
slides_color_val = color_elems[-2].get('val')
if slides_color_val is None:
return 0
elif color == "red" and not is_red(slides_color_val):
return 0
elif color == "blue" and not is_blue(slides_color_val):
return 0
elif color == "green" and not is_green(slides_color_val):
return 0
elif color == "black" and not is_black(slides_color_val):
return 0
return 1

60
evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json
View File

@@ -3,10 +3,66 @@
"snapshot": "libreoffice_impress",
"instruction": "I am preparing a PPT in Libreoffice impress. The slide number is barely visible to me. Please help me change the color of the slide number to red?",
"source": "https://superuser.com/questions/1674211/how-to-change-colour-of-slide-number-in-libre-office",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1_AmuPiTAyCSrquI2ERV0DnUtt1yaxtcj&export=download&authuser=0&confirm=t&uuid=d9c63ec2-45e6-4198-8519-c04c497ea1bb&at=APZUnTVtpEJatGaAex2OPSHDWgGM:1706533369423",
"path": "Desktop/saa-format-guide.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/saa-format-guide.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "saa-format-guide.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_page_number_colors",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/saa-format-guide.pptx",
"dest": "saa-format-guide.pptx"
}
}
}