From 5b5475094e94b6877e2df43ec83be64b1a0b8000 Mon Sep 17 00:00:00 2001 From: tsuky_chen <3107760494@qq.com> Date: Thu, 7 Mar 2024 01:24:36 +0800 Subject: [PATCH] update multi apps --- desktop_env/evaluators/metrics/__init__.py | 3 +- desktop_env/evaluators/metrics/chrome.py | 11 ++++ desktop_env/evaluators/metrics/docs.py | 10 +++ desktop_env/evaluators/metrics/slides.py | 4 +- .../02ce9a50-7af2-47ed-8596-af0c230501f8.json | 61 +++++++++++++++++++ .../09a37c51-e625-49f4-a514-20a773797a8a.json | 48 +++++++++++++++ .../4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json | 44 +++++++++++++ .../a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json | 42 +++++++++++-- 8 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json create mode 100644 evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json create mode 100644 evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index c965a95..5906b99 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -47,7 +47,8 @@ from .docs import ( check_file_exists, check_tabstops, compare_contains_image, - compare_docx_images + compare_docx_images, + compare_image_text ) from .general import ( check_csv, diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index 3c367b3..5018f5c 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -91,6 +91,17 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float: bookmark_bar_websites_urls = [bookmark['url'] for bookmark in bookmarks['bookmark_bar']['children'] if bookmark['type'] == 'url'] return 1. if set(bookmark_bar_websites_urls) == set(rule['urls']) else 0. + elif rule['type'] == "liked_authors_websites_urls": + # Check if "liked authors" folder exists + liked_authors_folder = next((bookmark for bookmark in bookmarks['bookmark_bar']['children'] if + bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None) + if liked_authors_folder: + # Check if it contains the specified URLs + liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if + bookmark['type'] == 'url'] + return 1. if set(liked_authors_urls) == set(rule['urls']) else 0. + else: + return 0. else: raise TypeError(f"{rule['type']} not support yet!") diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index b44df9c..613579a 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -183,6 +183,16 @@ def compare_docx_images(docx_file1, docx_file2): return 0 return 1 +import pytesseract + +def compare_image_text(image_path, rule): + img = Image.open(image_path) + img_text = pytesseract.image_to_string(img) + if rule['type'] == 'text': + return 1 if rule['text'] in img_text else 0 + else: + raise ValueError("Unsupported rule type") + def compare_line_spacing(docx_file1, docx_file2): if not compare_docx_files(docx_file1, docx_file2): return 0 diff --git a/desktop_env/evaluators/metrics/slides.py b/desktop_env/evaluators/metrics/slides.py index 9bbfb38..76217e1 100644 --- a/desktop_env/evaluators/metrics/slides.py +++ b/desktop_env/evaluators/metrics/slides.py @@ -182,7 +182,7 @@ def compare_pptx_files(file1_path, file2_path, **options): else: return None - if get_slide_notes(slide1) != get_slide_notes(slide2) and examine_note: + if get_slide_notes(slide1).strip() != get_slide_notes(slide2).strip() and examine_note: return 0 # check if the shapes are the same for shape1, shape2 in zip(slide1.shapes, slide2.shapes): @@ -235,7 +235,7 @@ def compare_pptx_files(file1_path, file2_path, **options): return 0 if hasattr(shape1, "text") and hasattr(shape2, "text"): - if shape1.text != shape2.text and examine_text: + if shape1.text.strip() != shape2.text.strip() and examine_text: return 0 # check if the paragraphs are the same diff --git a/evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json b/evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json new file mode 100644 index 0000000..092a706 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json @@ -0,0 +1,61 @@ +{ + "id": "02ce9a50-7af2-47ed-8596-af0c230501f8", + "snapshot": "libreoffice_writer", + "instruction": "I'm using libreoffice writer to write a tutorial about linux, and now I want to show the results obtained by using the \"ls\" command in /home/user. Please run this command and save the screenshot as 'ls.png' on Desktop.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1cOYh50n0S3-Q_h8HDTCEPv2v6NsJl07m&export=download&authuser=0&confirm=t&uuid=9741b52a-d998-4875-94ec-ab0a45a40bf9&at=APZUnTX__WVALT4i0EIQoRAWkm-k:1709715620836", + "path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/top-10-linux-commands-for-newbies.docx" + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('f11'); time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.scroll(-20)" + ] + } + } + ], + "trajectory": "trajectories/02ce9a50-7af2-47ed-8596-af0c230501f8", + "related_apps": [ + "libreoffice_writer", + "os" + ], + "evaluator": { + "func": "compare_image_text", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/ls.png", + "dest": "ls.png" + }, + "expected": { + "type": "rule", + "rules": { + "type": "text", + "text": "$ ls\n" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json b/evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json new file mode 100644 index 0000000..30c04b1 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json @@ -0,0 +1,48 @@ +{ + "id": "09a37c51-e625-49f4-a514-20a773797a8a", + "snapshot": "libreoffice_writer", + "instruction": "I received a request from my friend that he wanted me to help him modify a picture. On the Desktop is the requirement doc and the picture to be adjusted. Modify the image as he said and save modified pic as \"pic.jpg\" on Desktop. Thanks!", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1npPOtBdE5t_mzScyA94vDlxB8SCZNubv&export=download&authuser=0&confirm=t&uuid=64ee33b7-66a5-4f8e-9c98-95bea9521504&at=APZUnTW74Ntotdg-xRjaoxuDFju9:1709727407282", + "path": "/home/user/Desktop/requirment.docx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1eQHixX0TTExye0lbmHQny5CYhaREB5fQ&export=download&authuser=0&confirm=t&uuid=8546ef19-ea50-4e1a-9296-2db0302afd62&at=APZUnTVEmYOeAIdok_anTWOVHBCp:1709727527585", + "path": "/home/user/Desktop/ChMkKV8wsR6IBfEtABYfc0Tgu9cAAA1lQHO_78AFh-L733.jpg" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/requirment.docx" + } + } + ], + "trajectory": "trajectories/09a37c51-e625-49f4-a514-20a773797a8a", + "related_apps": [ + "libreoffice_writer", + "gimp", + "os" + ], + "evaluator": { + "func": "compare_images", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803", + "dest": "pic.jpg" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/pic.jpg", + "dest": "pic.jpg" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json b/evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json new file mode 100644 index 0000000..ca06a97 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json @@ -0,0 +1,44 @@ +{ + "id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487", + "snapshot": "libreoffice_impress", + "instruction": "I found the image on the second slide is too dark. Could you make it brighter for me? Keep the modified picture on Desktop. Name it \"background\". Thanks!", + "source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1SD5GlVUuG9RijMCEzuniPq1dIWxfr8xQ&export=download&authuser=0&confirm=t&uuid=67c13e54-9368-4e77-bda9-31776ec37479&at=APZUnTXGR14_2pztP7HoAiELiztJ:1709709598471", + "path": "/home/user/Desktop/PPT-Template_widescreen.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/PPT-Template_widescreen.pptx" + } + } + ], + "trajectory": "trajectories/4c26e3f3-3a14-4d86-b44a-d3cedebbb487", + "related_apps": [ + "gimp", + "libreoffice_impress", + "os" + ], + "evaluator": { + "func": "check_brightness_decrease_and_structure_sim", + "expected": { + "type": "vm_file", + "path": "/home/user/Desktop/background.png", + "dest": "background.png" + }, + "result": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1lpRSXEoZq3ENOG5ekaAsBQSNv5ig0mDr&export=download&authuser=0&confirm=t&uuid=4cb10a33-81b3-4814-a969-f469832e33e5&at=APZUnTWN3pyiVpS003vLOgCcq2gu:1709710047375", + "dest": "image_original.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json b/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json index 63cd0d5..504e5ce 100644 --- a/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json +++ b/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json @@ -4,6 +4,25 @@ "instruction": "I'm really enjoying this paper. Could you please find the personal webpages of the first author and the last three authors, and add them to a browser bookmark folder named 'Liked Authors'?", "source": "authors", "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, { "type": "download", "parameters": { @@ -40,15 +59,28 @@ ], "trajectory": "trajectories/a82b78bb-7fde-4cb3-94a4-035baf10bcf0", "related_apps": [ + "chrome", + "pdf" ], "evaluator": { - "postconfig": [], - "func": "", + "func": "is_expected_bookmarks", "result": { + "type": "bookmarks" }, "expected": { - }, - "options": { + "type": "rule", + "rules": { + "type": "liked_authors_websites_urls", + "names": [ + "Liked Authors" + ], + "urls": [ + "https://jimfan.me/", + "https://ai.stanford.edu/~dahuang/", + "https://yukezhu.me/", + "https://www.eas.caltech.edu/people/anima" + ] + } } } -} +} \ No newline at end of file