diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 0b388e5..bb30d12 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,13 +1,25 @@ -from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, \ - get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \ - get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info +from .chrome import ( + get_default_search_engine, + get_cookie_data, + get_bookmarks, + get_open_tabs_info, + get_pdf_from_url, + get_shortcuts_on_desktop, + get_history, + get_enabled_experiments, + get_chrome_language, + get_chrome_font_size, + get_profile_name, + get_number_of_search_results, + get_googledrive_file, + get_active_tab_info +) from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line, get_vm_terminal_output +from .gimp import get_gimp_config_file from .impress import get_audio_in_slide from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config from .vscode import get_vscode_config -from .impress import get_audio_in_slide -from .gimp import get_gimp_config_file diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index c4b3c6f..82bce50 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -1,26 +1,58 @@ -from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_cookie_deleted, is_shortcut_on_desktop, \ - check_font_size, \ - check_enabled_experiments, check_history_deleted, is_expected_search_query -from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines -from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \ - evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \ - check_no_duplicates, compare_init_lines -from .general import exact_match, fuzzy_match - -from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \ - compare_insert_equation, compare_highlighted_text -from .docs import is_first_line_centered, check_file_exists, compare_contains_image -from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json, check_list -from .general import exact_match, fuzzy_match, check_include_exclude -from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions -from .gimp import compare_images -from .libreoffice import check_libre_locale -from .pdf import check_pdf_pages -from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \ - compare_pptx_files, check_strikethrough, \ - check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel, check_transition -from .table import compare_table, compare_csv -from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter +from .basic_os import ( + check_gnome_favorite_apps, + is_utc_0, + check_text_enlarged, + check_moved_jpgs +) +from .chrome import ( + is_expected_tabs, + is_expected_bookmarks, + compare_pdfs, + is_cookie_deleted, + is_shortcut_on_desktop, + check_font_size, + check_enabled_experiments, + check_history_deleted, + is_expected_search_query +) +from .docs import ( + compare_font_names, + compare_subscript_contains, + has_page_numbers_in_footers, + compare_docx_lines, + evaluate_colored_words_in_tables, + check_highlighted_words, + evaluate_strike_through_last_paragraph, + evaluate_conversion, + evaluate_spacing, + check_italic_font_size_14, + evaluate_alignment, + get_unique_train_ids, + check_no_duplicates, + compare_init_lines +) +from .docs import ( + find_default_font, + contains_page_break, + compare_docx_files, + compare_docx_tables, + compare_line_spacing, + compare_insert_equation, + compare_highlighted_text, + is_first_line_centered, + check_file_exists, + compare_contains_image +) +from .general import ( + check_csv, + check_accessibility_tree, + run_sqlite3, + check_json, + check_list, + exact_match, + fuzzy_match, + check_include_exclude +) from .gimp import ( check_brightness_decrease_and_structure_sim, check_contrast_increase_and_structure_sim, @@ -34,11 +66,53 @@ from .gimp import ( check_triangle_position, check_structure_sim, check_config_status, + compare_images, + increase_saturation, + decrease_brightness, + check_file_exists, + compare_triangle_positions ) -from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ - compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \ - check_qt_max_volume, \ - check_qt_slider_colours, check_global_key_play_pause -from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \ +from .libreoffice import check_libre_locale +from .pdf import check_pdf_pages +from .slides import ( + check_presenter_console_disable, + check_image_stretch_and_center, + check_slide_numbers_color, + compare_pptx_files, + check_strikethrough, + check_slide_orientation_Portrait, + evaluate_presentation_fill_to_rgb_distance, + check_left_panel, + check_transition, + check_page_number_colors +) +from .table import ( + compare_table, + compare_csv +) +from .thunderbird import ( + check_thunderbird_prefs, + check_thunderbird_filter +) +from .vlc import ( + is_vlc_playing, + is_vlc_recordings_folder, + is_vlc_fullscreen, + compare_images, + compare_audios, + compare_videos, + check_qt_bgcone, + check_one_instance_when_started_from_file, + check_qt_minimal_view, + check_qt_max_volume, + check_qt_slider_colours, + check_global_key_play_pause +) +from .vscode import ( + compare_text_file, + compare_config, + compare_answer, + is_extension_installed, + check_json_settings, check_json_keybindings -from .basic_os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs +) diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index e66e63f..466483b 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -2,6 +2,7 @@ import logging import os import re import xml.etree.ElementTree as ET +import zipfile from typing import List, Dict, Any from docx import Document @@ -10,7 +11,6 @@ from docx.shared import RGBColor from odf.opendocument import load from odf.text import P from odf.text import Span -import zipfile logger = logging.getLogger("desktopenv.metric.docs") diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 8269892..b513260 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -211,7 +211,10 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path): img_tgt_normalized = normalize_brightness(img_tgt, target_brightness) structure_same = structure_check_by_mse(img_src_normalized, img_tgt_normalized) - return brightness_reduced and structure_same + if brightness_reduced and structure_same: + return 1. + else: + return 0. def check_saturation_increase_and_structure_sim(src_path, tgt_path): @@ -240,7 +243,10 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path): else: structure_same = False - return saturation_increased and structure_same + if saturation_increased and structure_same: + return 1. + else: + return 0. def check_file_exists_and_structure_sim(src_path, tgt_path): @@ -295,7 +301,10 @@ def check_triangle_position(tgt_path): tolerance = 0.05 * np.array(img_array.shape[:2]) middle = np.all(np.abs(centroid - image_center) < tolerance) - return middle + if bool(middle): + return 1. + else: + return 0. def check_structure_sim(src_path, tgt_path): @@ -326,7 +335,10 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path): # Check structure structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.65) - return higher_contrast and structure_same + if higher_contrast and structure_same: + return 1. + else: + return 0. def check_config_status(actual_config_path, rule): @@ -342,13 +354,13 @@ def check_config_status(actual_config_path, rule): items = line.strip().lstrip('(').rstrip(')\n').split() if isinstance(rule["key"], str): if items[0] == rule["key"] and items[-1] == rule["value"]: - return True + return 1. elif isinstance(rule["key"], list) and len(rule["key"]) == 2: if items[0] == rule["key"][0] \ and items[1] == rule["key"][1] \ and items[-1] == rule["value"]: - return True - return False + return 1. + return 0. def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None): @@ -374,7 +386,10 @@ def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=Non resized_target_image = target_image.resize(source_image.size) structure_same = structure_check_by_ssim(source_image, resized_target_image) - return width_same and height_same and structure_same + if width_same and height_same and structure_same: + return 1. + else: + return 0. def check_palette_and_structure_sim(src_path, tgt_path): @@ -390,7 +405,10 @@ def check_palette_and_structure_sim(src_path, tgt_path): target_image = Image.open(tgt_path) source_image = source_image.convert('RGB') structure_same = structure_check_by_ssim(source_image, target_image) - return palette_based and structure_same + if palette_based and structure_same: + return 1. + else: + return 0. def check_textbox_on_leftside(src_path): @@ -412,7 +430,10 @@ def check_textbox_on_leftside(src_path): break # Stop after finding the first dark pixel in this row # Here we define "almost" on the left side as being within the left 5% of the image - return left_most_dark_pixel < width * 0.05 + if left_most_dark_pixel < width * 0.05: + return 1. + else: + return 0. def check_image_mirror(src_path, tgt_path): diff --git a/desktop_env/evaluators/metrics/slides.py b/desktop_env/evaluators/metrics/slides.py index b7024a2..d5888b0 100644 --- a/desktop_env/evaluators/metrics/slides.py +++ b/desktop_env/evaluators/metrics/slides.py @@ -318,3 +318,50 @@ def check_transition(pptx_file, rules): return 0. else: return 0. + + +def check_page_number_colors(pptx_file, rules): + color = rules["color"] + + def is_red(rgb_str, threshold=50): + r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16) + return r > g + threshold and r > b + threshold + + def is_blue(rgb_str, threshold=50): + r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16) + return b > g + threshold and b > r + threshold + + def is_green(rgb_str, threshold=50): + r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16) + return g > r + threshold and g > b + threshold + + def is_black(rgb_str, threshold=50): + r, g, b = int(rgb_str[1:3], 16), int(rgb_str[3:5], 16), int(rgb_str[5:7], 16) + return r < threshold and g < threshold and b < threshold + + with zipfile.ZipFile(pptx_file, 'r') as zip_ref: + slide_master_name = 'ppt/slideMasters/slideMaster1.xml' + with zip_ref.open(slide_master_name) as slide_master_file: + tree = ET.parse(slide_master_file) + root = tree.getroot() + + namespaces = { + 'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', + 'p': 'http://schemas.openxmlformats.org/presentationml/2006/main', + } + + color_elems = root.findall('.//a:solidFill//a:srgbClr', namespaces) + slides_color_val = color_elems[-2].get('val') + + if slides_color_val is None: + return 0 + elif color == "red" and not is_red(slides_color_val): + return 0 + elif color == "blue" and not is_blue(slides_color_val): + return 0 + elif color == "green" and not is_green(slides_color_val): + return 0 + elif color == "black" and not is_black(slides_color_val): + return 0 + + return 1 diff --git a/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json b/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json index 3f465a2..2e83734 100644 --- a/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json +++ b/evaluation_examples/examples/libreoffice_impress/ac9bb6cb-1888-43ab-81e4-a98a547918cd.json @@ -3,10 +3,66 @@ "snapshot": "libreoffice_impress", "instruction": "I am preparing a PPT in Libreoffice impress. The slide number is barely visible to me. Please help me change the color of the slide number to red?", "source": "https://superuser.com/questions/1674211/how-to-change-colour-of-slide-number-in-libre-office", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1_AmuPiTAyCSrquI2ERV0DnUtt1yaxtcj&export=download&authuser=0&confirm=t&uuid=d9c63ec2-45e6-4198-8519-c04c497ea1bb&at=APZUnTVtpEJatGaAex2OPSHDWgGM:1706533369423", + "path": "Desktop/saa-format-guide.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/saa-format-guide.pptx" + } + } + ], "trajectory": "trajectories/", "related_apps": [ "libreoffice_impress" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "saa-format-guide.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "check_page_number_colors", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/saa-format-guide.pptx", + "dest": "saa-format-guide.pptx" + } + } }