chrome (google drive related) + X multi-app examples finished (leaving two emails and thunderbird-profile.tar.gz to be crafted)

desktop_env/evaluators/metrics/__init__.py

@@ -13,6 +13,7 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
 from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json
 from .general import exact_match, fuzzy_match, check_include_exclude
 from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
+from .gimp import compare_images
 from .libreoffice import check_libre_locale
 from .pdf import check_pdf_pages
 from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \

desktop_env/evaluators/metrics/gimp.py

@@ -1,6 +1,22 @@
 import os
+from typing import List, Union
 from PIL import Image, ImageChops, ImageStat
 
+def compare_images(pred_img_path_list: Union[str, List[str]], gold_img_path_list: Union[str, List[str]]) -> float:
+    """ Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0
+    """
+    if type(pred_img_path_list) != list:
+        pred_img_path_list = [pred_img_path_list]
+        gold_img_path_list = [gold_img_path_list]
+    for pred_img_path, gold_img_path in zip(pred_img_path_list, gold_img_path_list):
+        pred_img = Image.open(pred_img_path)
+        gold_img = Image.open(gold_img_path)
+        diff = ImageChops.difference(pred_img, gold_img)
+        if diff.getbbox():
+            return 0.0
+    return 1.0
+
+
 def get_gimp_export_path():
     # Path to GIMP's configuration file. This example assumes GIMP version 2.10.
     # You need to adjust the path according to the GIMP version and user's file system.

evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json

@@ -1,7 +1,7 @@
 {
     "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
     "snapshot": "chrome",
-    "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).",
+    "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
     "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
     "config": [
         {
@@ -93,7 +93,7 @@
         "chrome"
     ],
     "evaluator": {
-        "func": "compare_figures",
+        "func": "compare_images",
         "result": {
             "type": "googledrive_file",
             "settings_file": "evaluation_examples/settings/googledrive/settings.yml",
@@ -120,9 +120,9 @@
         "expected": {
             "type": "cloud_file",
             "path": [
-                "file1",
+                "https://drive.usercontent.google.com/download?id=19J5tzWjx9hdo-n0MC3upzAntVMa8WUgk&export=download&authuser=0&confirm=t&uuid=be790579-8db9-4bd2-a757-beb27af386af&at=APZUnTVM2PjNDXhlwFZ6WAFdNVsD:1706497547717",
-                "file2",
+                "https://drive.usercontent.google.com/download?id=1S04RpR5dk80LylIYGvA4e3sAUBd6wdlQ&export=download&authuser=0&confirm=t&uuid=b302de03-04f7-455c-ab0c-b3cbbeb6929a&at=APZUnTVD8zMZGO1_GWaFUm1cNXul:1706497555463",
-                "file3"
+                "https://drive.usercontent.google.com/download?id=11NRLh93RTzEd0Cy-cYwMyNJSFG7-vP9c&export=download&authuser=0&confirm=t&uuid=02500115-dea3-481a-af4f-a723d9a62169&at=APZUnTW9-gENlsyfdIPA4PTA0emh:1706497560874"
             ],
             "dest": [
                 "1_gold.png",