lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chrome (google drive related) + X multi-app examples finished (leaving two emails and thunderbird-profile.tar.gz to be crafted)

Browse Source
This commit is contained in:
rhythmcao
2024-01-29 11:16:27 +08:00
parent cc21c3a6b1
commit 532826835d
3 changed files with 22 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
desktop_env/evaluators/metrics/__init__.py
View File

@@ -13,6 +13,7 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
from .general import check_csv, check_accessibility_tree, run_sqlite3, check_json
from .general import exact_match, fuzzy_match, check_include_exclude
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .gimp import compare_images
from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \

16
desktop_env/evaluators/metrics/gimp.py
View File

@@ -1,6 +1,22 @@
import os
from typing import List, Union
from PIL import Image, ImageChops, ImageStat
def compare_images(pred_img_path_list: Union[str, List[str]], gold_img_path_list: Union[str, List[str]]) -> float:
""" Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0
"""
if type(pred_img_path_list) != list:
pred_img_path_list = [pred_img_path_list]
gold_img_path_list = [gold_img_path_list]
for pred_img_path, gold_img_path in zip(pred_img_path_list, gold_img_path_list):
pred_img = Image.open(pred_img_path)
gold_img = Image.open(gold_img_path)
diff = ImageChops.difference(pred_img, gold_img)
if diff.getbbox():
return 0.0
return 1.0
def get_gimp_export_path():
# Path to GIMP's configuration file. This example assumes GIMP version 2.10.
# You need to adjust the path according to the GIMP version and user's file system.

10
evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -1,7 +1,7 @@
{
"id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
"snapshot": "chrome",
"instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).",
"instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
@@ -93,7 +93,7 @@
"chrome"
],
"evaluator": {
"func": "compare_figures",
"func": "compare_images",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
@@ -120,9 +120,9 @@
"expected": {
"type": "cloud_file",
"path": [
"file1",
"file2",
"file3"
"https://drive.usercontent.google.com/download?id=19J5tzWjx9hdo-n0MC3upzAntVMa8WUgk&export=download&authuser=0&confirm=t&uuid=be790579-8db9-4bd2-a757-beb27af386af&at=APZUnTVM2PjNDXhlwFZ6WAFdNVsD:1706497547717",
"https://drive.usercontent.google.com/download?id=1S04RpR5dk80LylIYGvA4e3sAUBd6wdlQ&export=download&authuser=0&confirm=t&uuid=b302de03-04f7-455c-ab0c-b3cbbeb6929a&at=APZUnTVD8zMZGO1_GWaFUm1cNXul:1706497555463",
"https://drive.usercontent.google.com/download?id=11NRLh93RTzEd0Cy-cYwMyNJSFG7-vP9c&export=download&authuser=0&confirm=t&uuid=02500115-dea3-481a-af4f-a723d9a62169&at=APZUnTW9-gENlsyfdIPA4PTA0emh:1706497560874"
],
"dest": [
"1_gold.png",