diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 82b889e..8bd1cdc 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -13,6 +13,7 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json from .general import exact_match, fuzzy_match, check_include_exclude from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions +from .gimp import compare_images from .libreoffice import check_libre_locale from .pdf import check_pdf_pages from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \ diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 1e919ef..6d1a30e 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -1,6 +1,22 @@ import os +from typing import List, Union from PIL import Image, ImageChops, ImageStat +def compare_images(pred_img_path_list: Union[str, List[str]], gold_img_path_list: Union[str, List[str]]) -> float: + """ Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0 + """ + if type(pred_img_path_list) != list: + pred_img_path_list = [pred_img_path_list] + gold_img_path_list = [gold_img_path_list] + for pred_img_path, gold_img_path in zip(pred_img_path_list, gold_img_path_list): + pred_img = Image.open(pred_img_path) + gold_img = Image.open(gold_img_path) + diff = ImageChops.difference(pred_img, gold_img) + if diff.getbbox(): + return 0.0 + return 1.0 + + def get_gimp_export_path(): # Path to GIMP's configuration file. This example assumes GIMP version 2.10. # You need to adjust the path according to the GIMP version and user's file system. diff --git a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json index 5475c70..0bf4bcb 100644 --- a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json +++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -1,7 +1,7 @@ { "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b", "snapshot": "chrome", - "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).", + "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).", "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", "config": [ { @@ -93,7 +93,7 @@ "chrome" ], "evaluator": { - "func": "compare_figures", + "func": "compare_images", "result": { "type": "googledrive_file", "settings_file": "evaluation_examples/settings/googledrive/settings.yml", @@ -120,9 +120,9 @@ "expected": { "type": "cloud_file", "path": [ - "file1", - "file2", - "file3" + "https://drive.usercontent.google.com/download?id=19J5tzWjx9hdo-n0MC3upzAntVMa8WUgk&export=download&authuser=0&confirm=t&uuid=be790579-8db9-4bd2-a757-beb27af386af&at=APZUnTVM2PjNDXhlwFZ6WAFdNVsD:1706497547717", + "https://drive.usercontent.google.com/download?id=1S04RpR5dk80LylIYGvA4e3sAUBd6wdlQ&export=download&authuser=0&confirm=t&uuid=b302de03-04f7-455c-ab0c-b3cbbeb6929a&at=APZUnTVD8zMZGO1_GWaFUm1cNXul:1706497555463", + "https://drive.usercontent.google.com/download?id=11NRLh93RTzEd0Cy-cYwMyNJSFG7-vP9c&export=download&authuser=0&confirm=t&uuid=02500115-dea3-481a-af4f-a723d9a62169&at=APZUnTW9-gENlsyfdIPA4PTA0emh:1706497560874" ], "dest": [ "1_gold.png",