From 2c08a02206dea4908ce54f5d7aa4e1a746bb85c8 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 27 Feb 2024 18:37:32 +0800 Subject: [PATCH 1/9] fix the error caused by url encoding --- desktop_env/evaluators/getters/chrome.py | 10 +++++++--- .../chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json | 4 ++-- .../chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index d40de23..48060a6 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -4,6 +4,7 @@ import os import platform import sqlite3 import time +from urllib.parse import unquote from typing import Dict, Any, List from urllib.parse import urlparse, parse_qs @@ -985,6 +986,7 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]): """ active_tab_url = get_active_url_from_accessTree(env, config) if not isinstance(active_tab_url, str): + logger.error("active_tab_url is not a string") return None host = env.vm_ip port = 9222 # fixme: this port is hard-coded, need to be changed from config file @@ -1017,12 +1019,14 @@ def get_active_tab_html_parse(env, config: Dict[str, Any]): for context in browser.contexts: for page in context.pages: page.wait_for_load_state("networkidle") - if page.url == active_tab_url: + # the accTree and playwright can get encoding(percent-encoding) characters, we need to convert them to normal characters + if unquote(page.url) == unquote(active_tab_url): target_page = page - print("tartget page url: ", target_page.url) - print("tartget page title: ", target_page.title()) + print("\33[32mtartget page url: ", target_page.url, "\33[0m") + print("\33[32mtartget page title: ", target_page.title(), "\33[0m") break if target_page is None: + logger.error("Your tab is not the target tab.") return {} return_json = {} if config["category"] == "class": diff --git a/evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json b/evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json index 566d666..e545c9b 100644 --- a/evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json +++ b/evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json @@ -62,8 +62,8 @@ "type": "rule", "rules":{ "expected": { - "locationName": "Zurich Airport", - "dropLocationName": "Zurich Airport", + "locationName": "Zürich", + "dropLocationName": "Zürich", "filterCriteria_carCategory": "large", "filterCriteria_sortBy": "PRICE" } diff --git a/evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json b/evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json index 6f37745..b2a6b2a 100644 --- a/evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json +++ b/evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json @@ -54,7 +54,7 @@ "rules":{ "expected": { "q": "drip coffee maker", - "tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,pdtr0:1825161|1825162" + "tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,sales:1,pdtr0:1825161|1825162" } } } From 5ebd080237898843157b8b77263781faff354c67 Mon Sep 17 00:00:00 2001 From: BlankCheng <913501223@qq.com> Date: Wed, 6 Mar 2024 21:33:38 +0800 Subject: [PATCH 2/9] Update multi-app examples --- desktop_env/evaluators/metrics/__init__.py | 10 +- desktop_env/evaluators/metrics/gimp.py | 102 ++++++++++++++---- desktop_env/evaluators/metrics/vscode.py | 51 +++++++++ .../d16c99dc-2a1e-46f2-b350-d97c86c85c15.json | 32 ++++-- .../3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json | 37 +++++++ .../42f4d1c7-4521-4161-b646-0a8934e36081.json | 71 ++++++++++++ .../7f35355e-02a6-45b5-b140-f0be698bcf85.json | 43 ++++++++ .../91190194-f406-4cd6-b3f9-c43fac942b22.json | 51 +++++++++ .../98e8e339-5f91-4ed2-b2b2-12647cb134f4.json | 54 ++++++++++ .../bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json | 69 ++++++++++++ .../d68204bf-11c1-4b13-b48b-d303c73d4bf6.json | 51 +++++++++ .../e8172110-ec08-421b-a6f5-842e6451911f.json | 67 ++++++++++++ 12 files changed, 606 insertions(+), 32 deletions(-) create mode 100644 evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json create mode 100644 evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json create mode 100644 evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json create mode 100644 evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json create mode 100644 evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json create mode 100644 evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json create mode 100644 evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json create mode 100644 evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 527dbe5..6d49742 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -64,7 +64,7 @@ from .gimp import ( check_brightness_decrease_and_structure_sim, check_contrast_increase_and_structure_sim, check_saturation_increase_and_structure_sim, - check_image_size_and_structure_sim, + check_image_size, check_image_mirror, check_palette_and_structure_sim, check_textbox_on_leftside, @@ -77,7 +77,9 @@ from .gimp import ( increase_saturation, decrease_brightness, check_file_exists, - compare_triangle_positions + compare_triangle_positions, + check_sharper, + check_image_file_size ) from .libreoffice import check_libre_locale from .pdf import check_pdf_pages @@ -121,11 +123,13 @@ from .vscode import ( compare_text_file, compare_config, compare_answer, + compare_result_files, is_extension_installed, check_json_settings, check_json_keybindings, check_python_file_by_test_suite, - check_python_file_by_gold_file + check_python_file_by_gold_file, + check_html_background_image ) diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 30c9b68..e46f5d5 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -5,7 +5,7 @@ from PIL import Image, ImageChops, ImageStat def compare_image_list(pred_img_path_list: Union[str, List[str]], - gold_img_path_list: Union[str, List[str]]) -> float: + gold_img_path_list: Union[str, List[str]]) -> float: """ Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0 """ if type(pred_img_path_list) != list: @@ -177,6 +177,16 @@ def calculate_contrast(image): return np.std(pixels) +def calculate_image_sharpness(image_path): + # Load the image in grayscale + image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE) + # Apply the Laplacian operator + laplacian = cv2.Laplacian(image, cv2.CV_64F) + # Calculate the variance + variance = np.var(laplacian) + return variance + + def structure_check_by_mse(img1, img2, threshold=0.03): """Check if two images are approximately the same by MSE""" mse = np.mean( @@ -295,7 +305,8 @@ def check_triangle_position(tgt_path): # We assume the triangle is a different color from the background # Find the unique colors - unique_colors, counts = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0, return_counts=True) + unique_colors, counts = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0, + return_counts=True) unique_colors_sorted = unique_colors[np.argsort(counts)] # Assuming the background is the most common color and the triangle is a different color @@ -337,6 +348,25 @@ def check_structure_sim(src_path, tgt_path): return structure_same +def check_structure_sim_resized(src_path, tgt_path): + """ + Check if the structure of the two images are similar after resizing. + gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15 + """ + if src_path is None or tgt_path is None: + return 0. + + img_src = Image.open(src_path) + img_tgt = Image.open(tgt_path) + + # Resize the images to the same size + img_src = img_src.resize(img_tgt.size) + + # Check if the structure is similar + structure_same = structure_check_by_ssim(img_src, img_tgt) + return structure_same + + def check_contrast_increase_and_structure_sim(src_path, tgt_path): """ Check if the src image has higher contrast than the tgt image and the structures are similar @@ -388,34 +418,28 @@ def check_config_status(actual_config_path, rule): return 0. -def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None): +def check_image_size(src_path, rule): """ - Check if the size of the src image is correct and the structure of the two images are similar. - gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15 + Check if the size of the src image is correct + multi-apps:42f4d1c7-4521-4161-b646-0a8934e36081 """ - - if src_path is None or tgt_path is None: + if src_path is None: return 0. - # Load images - source_image = Image.open(src_path) - target_image = Image.open(tgt_path) + # Load the image + img = Image.open(src_path) - # Check size - if width is not None: - width_same = source_image.size[0] == width - else: - width_same = True - if height is not None: - height_same = source_image.size[1] == height + # Check the size + if rule["height"] is not None: + height_same = img.size[1] == rule["height"] else: height_same = True + if rule["width"] is not None: + width_same = img.size[0] == rule["width"] + else: + width_same = True - # Check structure - resized_target_image = target_image.resize(source_image.size) - structure_same = structure_check_by_ssim(source_image, resized_target_image) - - if width_same and height_same and structure_same: + if height_same and width_same: return 1. else: return 0. @@ -521,6 +545,31 @@ def check_green_background(src_path, tgt_path): return 1. +def check_sharper(src_path, tgt_path): + """ + Check if the source image is sharper than the target image. + multi-app:bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108 + """ + sharpness_src = calculate_image_sharpness(src_path) + sharpness_tgt = calculate_image_sharpness(tgt_path) + return 1.0 if sharpness_src > sharpness_tgt else 0.0 + + +def check_image_file_size(src_path, rule): + """ + Check if the size of the src image within 500KB + """ + if src_path is None: + return 0.0 + + # Check the size + file_size = os.path.getsize(src_path) + if file_size < rule["max_size"]: + return 1.0 + else: + return 0.0 + + if __name__ == "__main__": actual_config_path = "../../../cache/sessionrc_test" rule = { @@ -550,3 +599,12 @@ if __name__ == "__main__": tgt_path = "../../../cache/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce/Triangle_In_The_Middle.png" print(check_triangle_position(tgt_path)) + src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi_sharper.png" + tgt_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi.png" + print(check_sharper(src_path, tgt_path)) + + src_path = "../../../cache/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f/compressed.jpeg" + rule = { + "max_size": 500000 + } + print(check_image_file_size(src_path, rule)) \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index ecf4e10..2331a75 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -161,3 +161,54 @@ def check_python_file_by_test_suite(actual_files, test_file, **options) -> float def check_python_file_by_gold_file(actual_files, gold_file: str, **options) -> float: pass + + +def check_html_background_image(src_path: str, rule: Dict = None) -> float: + """ + Check if the background image is correctly set. + multi-app:bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108 + """ + from bs4 import BeautifulSoup + with open(src_path, 'r') as f: + html_content = f.read() + soup = BeautifulSoup(html_content, 'html.parser') + styles = soup.find_all('style') + for style in styles: + if f'background-image: url(\'{rule["value"]}\')' in style.text: + return 1.0 + return 0.0 + + +def compare_result_files(src_path, tgt_path): + """ + Compare whether the content of two files are the same. + multi-app:7f35355e-02a6-45b5-b140-f0be698bcf85 + """ + with open(src_path, 'r') as f: + src_content = f.read().strip() + with open(tgt_path, 'r') as f: + tgt_content = f.read().strip() + try: + # Compare the content as numbers + tgt_content_num = float(tgt_content) + if tgt_content in src_content: + # If the content of tgt is in src, return 1.0 since output src might be + # a superset(language description+number) of tgt + return 1.0 + src_content_num = float(src_content) + if abs(src_content_num - tgt_content_num) < 1e-4: + return 1.0 + return 0.0 + except: + if src_content == tgt_content: + return 1.0 + return 0.0 + + +if __name__ == "__main__": + src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/index.html" + rule = { + "type:": "value", + "value": "anmi_sharper.png" + } + print(check_html_background_image(src_path, rule)) diff --git a/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json b/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json index b5b1cf2..3029c0c 100644 --- a/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json +++ b/evaluation_examples/examples/gimp/d16c99dc-2a1e-46f2-b350-d97c86c85c15.json @@ -84,16 +84,34 @@ } } ], - "func": "check_image_size_and_structure_sim", - "expected":{ + "func": [ + "check_image_size", + "check_structure_sim" + ], + "expected": [ + { "type": "vm_file", "path": "/home/user/Desktop/dog_with_background.png", "dest": "dog_with_background.png" }, - "result": { - "type": "vm_file", - "path": "/home/user/Desktop/resized.png", - "dest": "resized.png" - } + { + "type": "vm_file", + "path": "/home/user/Desktop/dog_with_background.png", + "dest": "dog_with_background.png" + } + ], + "result": [ + { + "type": "rule", + "rules": { + "height": 512 + } + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/resized.png", + "dest": "resized.png" + } + ] } } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json b/evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json new file mode 100644 index 0000000..309f370 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f.json @@ -0,0 +1,37 @@ +{ + "id": "3c8f201a-009d-4bbe-8b65-a6f8b35bb57f", + "snapshot": "gimp", + "instruction": "Use `gdown` to download the image from \"https://drive.google.com/uc?export=download&id=1i8j5dGS57sA07jEuPNAlQW-sn5uqUnuK\", and then use GIMP to compress it to under 600KB. Resize if needed.", + "source": "", + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp", + "os" + ], + "evaluator": { + "func": "check_image_file_size", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/compressed.jpeg", + "dest": "compressed.jpeg" + }, + "expected": { + "type": "rule", + "rules": { + "max_size": 600000 + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json b/evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json new file mode 100644 index 0000000..a6adb54 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/42f4d1c7-4521-4161-b646-0a8934e36081.json @@ -0,0 +1,71 @@ +{ + "id": "42f4d1c7-4521-4161-b646-0a8934e36081", + "snapshot": "gimp", + "instruction": "Configure VS Code to edit GIMP script-fu scripts effectively by installing lisp extension. Test by writing code to resizing the image as 128 * 128 as \"resized.png\"", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1yrWU5HimYPNUjdtvw1a218kh50fPVtZ3", + "path": "/home/user/Desktop/character.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "code" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp", + "vs_code" + ], + "evaluator": { + "func": [ + "is_extension_installed", + "check_image_size" + ], + "result": [ + { + "type": "vm_command_line", + "command": [ + "code", + "--list-extensions", + "|", + "grep", + "mattn.lisp" + ] + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/resized.png", + "dest": "resized.png" + } + ], + "expected": [ + { + "type": "rule", + "rules": { + "type": "contain", + "expected": "mattn.lisp" + } + }, + { + "type": "rule", + "rules": { + "height": 128, + "width": 128 + } + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json b/evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json new file mode 100644 index 0000000..c33b042 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/7f35355e-02a6-45b5-b140-f0be698bcf85.json @@ -0,0 +1,43 @@ +{ + "id": "7f35355e-02a6-45b5-b140-f0be698bcf85", + "snapshot": "libreoffice_calc", + "instruction": "Export the table to a CSV file and then help me write code to find the medium price (fill empty value with average). Save the result in \"result.txt\".", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://docs.google.com/spreadsheets/d/13YL-KC__pav2qp3sFDs1BT2wZnpWGp7s/export?format=xlsx", + "path": "/home/user/Desktop/stock.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/stock.xlsx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_calc", + "vs_code" + ], + "evaluator": { + "func": "compare_result_files", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/result.txt", + "dest": "result.txt" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1oPPW_dozWGII5MRmdXdKKoEK5iBkd_8Q", + "dest": "result_gold.txt" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json b/evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json new file mode 100644 index 0000000..ba554e0 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/91190194-f406-4cd6-b3f9-c43fac942b22.json @@ -0,0 +1,51 @@ +{ + "id": "91190194-f406-4cd6-b3f9-c43fac942b22", + "snapshot": "gimp", + "instruction": "Launch GIMP from the command line to edit \"cola.png\" and crop the top 20% off the image for my avatar as \"cropped.png\".", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1bmSRNNh4JkF6izrKrmynUHarf0pFES50", + "path": "/home/user/Desktop/cola.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1MayrIPJWRK7cMEVe3TxYmgkAbVMrYcQA", + "path": "/home/user/Desktop/cropped_gold.png" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp", + "os" + ], + "evaluator": { + "func": "check_structure_sim", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/cropped.png", + "dest": "cropped.png" + }, + "expected": { + "type": "vm_file", + "path": "/home/user/Desktop/cropped_gold.png", + "dest": "cropped_gold.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json b/evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json new file mode 100644 index 0000000..3d08199 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/98e8e339-5f91-4ed2-b2b2-12647cb134f4.json @@ -0,0 +1,54 @@ +{ + "id": "98e8e339-5f91-4ed2-b2b2-12647cb134f4", + "snapshot": "vs_code", + "instruction": "Merge the contents of all .txt files from your vscode project into a single document in Writer. No merging separator is needed. Ensure to set the overall font size of the document to 10.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1dqbi6j3zPOqFTMFHVVaEOXSdHz5zIey1", + "path": "/home/user/Desktop/doc_proc.zip" + } + ] + } + }, + { + "type": "command", + "parameters": { + "command": "mkdir -p /home/user/Desktop/doc_proc/; unzip /home/user/Desktop/doc_proc.zip -d /home/user/Desktop/doc_proc/", + "shell": true + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "code", + "-g", + "/home/user/Desktop/doc_proc/" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vs_code", + "libreoffice_writer" + ], + "evaluator": { + "func": "compare_docx_files", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/concat.docx", + "dest": "concat.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=10k_8T7Hk9KQ2I-AbNK56hvXR6yv0Pemc", + "dest": "concat_gold.docx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json b/evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json new file mode 100644 index 0000000..7b1529e --- /dev/null +++ b/evaluation_examples/examples/multi_apps/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108.json @@ -0,0 +1,69 @@ +{ + "id": "bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108", + "snapshot": "gimp", + "instruction": "I have a background image named 'anmi.png'. I need the image to be enhanced for better sharpness, as 'anmi_sharper.png'. Next, please use it as the background image for my 'index.html' page.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1BAZvR0BflPh70ClL2bHrbJ-xyV-nNQ1o", + "path": "/home/user/Desktop/anmi.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1zcyY7y_deKStc-AtU8pFFrJItOuyfrNK", + "path": "/home/user/Desktop/index.html" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "/home/user/Desktop/anmi.png" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp", + "vs_code" + ], + "evaluator": { + "func": [ + "check_sharper", + "check_html_background_image" + ], + "result": [ + { + "type": "vm_file", + "path": "/home/user/Desktop/anmi_sharper.png", + "dest": "anmi_sharper.png" + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/index.html", + "dest": "index.html" + } + ], + "expected": [ + { + "type": "vm_file", + "path": "/home/user/Desktop/anmi.png", + "dest": "anmi.png" + }, + { + "type": "rule", + "rules": { + "type:": "value", + "value": "anmi_sharper.png" + } + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json b/evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json new file mode 100644 index 0000000..55af9af --- /dev/null +++ b/evaluation_examples/examples/multi_apps/d68204bf-11c1-4b13-b48b-d303c73d4bf6.json @@ -0,0 +1,51 @@ +{ + "id": "d68204bf-11c1-4b13-b48b-d303c73d4bf6", + "snapshot": "gimp", + "instruction": "Divide my image vertically into three equal sections with command line. Then rearrange them in order with a gradient of warm tones, progressively becoming warmer from left to right as a new image \"rearranged.png\".", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1CPGW_OZsfSWDdTU7CFrTjpzSAASyLy4w", + "path": "/home/user/Desktop/tilearray.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1aHwmnxL2CKEh_FhVpevY452-BQH2t5rG", + "path": "/home/user/Desktop/rearranged_gold.png" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"alt\", \"t\"); time.sleep(0.5);" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp", + "os" + ], + "evaluator": { + "func": "check_structure_sim", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/rearranged.png", + "dest": "rearranged.png" + }, + "expected": { + "type": "vm_file", + "path": "/home/user/Desktop/rearranged_gold.png", + "dest": "rearranged_gold.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json b/evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json new file mode 100644 index 0000000..b93696e --- /dev/null +++ b/evaluation_examples/examples/multi_apps/e8172110-ec08-421b-a6f5-842e6451911f.json @@ -0,0 +1,67 @@ +{ + "id": "e8172110-ec08-421b-a6f5-842e6451911f", + "snapshot": "gimp", + "instruction": "Open 'character.png' in GIMP and extract the pixel art character. Save the selected character as 'character_gimp.png'. Additionally, write a Python script to automate this selection process, ensuring it precisely mimics the manual extraction done in GIMP. Output the result from the script as 'character_code.png'.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1Tf7MvqBMHuIhvKmk-Ryr_qmzmkACFjxB", + "path": "/home/user/Desktop/character.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=1wKG5X6LaN0tShsAK4lFg3OyFg8OGYYZg", + "path": "/home/user/Desktop/character_no_background_gold.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "/home/user/Desktop/character.png" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp", + "vs_code" + ], + "evaluator": { + "func": [ + "check_structure_sim", + "check_structure_sim" + ], + "result": [ + { + "type": "vm_file", + "path": "/home/user/Desktop/character_gimp.png", + "dest": "character_gimp.png" + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/character_code.png", + "dest": "character_code.png" + } + ], + "expected": [ + { + "type": "vm_file", + "path": "/home/user/Desktop/character_no_background_gold.png", + "dest": "character_no_background_gold.png" + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/character_no_background_gold.png", + "dest": "character_no_background_gold.png" + } + ] + } +} \ No newline at end of file From 62fd8feebb41979c358bf90705457e602badae6e Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Fri, 8 Mar 2024 19:24:15 +0800 Subject: [PATCH 3/9] xiaochuan's multiapp examples --- desktop_env/evaluators/getters/__init__.py | 6 +- desktop_env/evaluators/getters/calc.py | 15 ++ desktop_env/evaluators/getters/chrome.py | 37 +++++ desktop_env/evaluators/getters/file.py | 17 +- desktop_env/evaluators/getters/misc.py | 7 + desktop_env/evaluators/metrics/__init__.py | 21 ++- desktop_env/evaluators/metrics/basic_os.py | 12 ++ desktop_env/evaluators/metrics/calc.py | 24 +++ desktop_env/evaluators/metrics/chrome.py | 6 + desktop_env/evaluators/metrics/demo.py | 28 ++++ desktop_env/evaluators/metrics/docs.py | 42 ++++- desktop_env/evaluators/metrics/general.py | 156 +++++++++++++++++- desktop_env/evaluators/metrics/utils.py | 2 + .../0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json | 25 ++- .../2ad9387a-65d8-4e33-ad5b-7580065a27ca.json | 2 +- .../b070486d-e161-459b-aa2b-ef442d973b92.json | 26 ++- .../cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json | 2 +- .../2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json | 49 ++++++ .../26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json | 69 ++++++++ .../3eb2a122-a5e3-4f89-9820-f7fa1a582969.json | 52 ++++++ .../5bc63fb9-276a-4439-a7c1-9dc76401737f.json | 70 ++++++++ .../65e62ec6-4603-4c67-b4d2-07830deb285b.json | 70 ++++++++ .../6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json | 54 ++++++ .../716a6079-22da-47f1-ba73-c9d58f986a38.json | 51 ++++++ .../7ff48d5b-2df2-49da-b500-a5150ffc7f18.json | 50 ++++++ .../873cafdd-a581-47f6-8b33-b9696ddb7b05.json | 66 ++++++++ .../9f3bb592-209d-43bc-bb47-d77d9df56504.json | 88 ++++++++++ .../a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json | 62 +++++++ .../ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json | 80 +++++++++ .../da922383-bfa4-4cd3-bbad-6bebab3d7742.json | 75 +++++++++ .../dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json | 53 ++++++ 31 files changed, 1286 insertions(+), 31 deletions(-) create mode 100644 desktop_env/evaluators/getters/calc.py create mode 100644 desktop_env/evaluators/metrics/calc.py create mode 100644 desktop_env/evaluators/metrics/demo.py create mode 100644 evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json create mode 100644 evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json create mode 100644 evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json create mode 100644 evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json create mode 100644 evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json create mode 100644 evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json create mode 100644 evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json create mode 100644 evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json create mode 100644 evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json create mode 100644 evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json create mode 100644 evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json create mode 100644 evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json create mode 100644 evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json create mode 100644 evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 43f4c1b..d78212b 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -23,14 +23,16 @@ from .chrome import ( get_active_tab_url_parse, get_gotoRecreationPage_and_get_html_content, get_url_dashPart, - get_active_url_from_accessTree + get_active_url_from_accessTree, + get_find_installed_extension_name ) from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line, get_vm_terminal_output from .gimp import get_gimp_config_file from .impress import get_audio_in_slide from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory -from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime +from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime, get_time_diff_range from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player from .vscode import get_vscode_config +from .calc import get_conference_city_in_order \ No newline at end of file diff --git a/desktop_env/evaluators/getters/calc.py b/desktop_env/evaluators/getters/calc.py new file mode 100644 index 0000000..7985231 --- /dev/null +++ b/desktop_env/evaluators/getters/calc.py @@ -0,0 +1,15 @@ +import csv + +# I want to write a function, reads a csv file, and get all the contents in the third column in the order of rows +def get_conference_city_in_order(env, config): + # read the csv file + csv_path = config['csv_path'] + print(f"Reading csv file from {csv_path}") + with open(csv_path, 'r') as f: + reader = csv.reader(f) + # skip the header row + next(reader) + # get the third column in the order of rows + conference_city_list = [row[2] for row in reader] + return conference_city_list + \ No newline at end of file diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 48060a6..962d8fe 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -919,6 +919,43 @@ def get_find_unpacked_extension_path(env, config: Dict[str, str]): return "Google" +def get_find_installed_extension_name(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ + 'output'].strip() + elif os_type == 'Linux': + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() + + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + # Preferences store all the path of installed extensions, return them all and let metrics try to find one matches the targeted extension path + all_extensions_name = [] + all_extensions = data.get('extensions', {}).get('settings', {}) + for id in all_extensions.keys(): + name = all_extensions[id]["manifest"]["name"] + all_extensions_name.append(name) + return all_extensions_name + except Exception as e: + logger.error(f"Error: {e}") + return "Google" + + def get_data_delete_automacally(env, config: Dict[str, str]): """ This function is used to open th "auto-delete" mode of chromium diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 95c0a18..72c9c54 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,7 +1,7 @@ import os from typing import Dict, List, Set from typing import Optional, Any, Union - +from datetime import datetime import requests @@ -58,14 +58,24 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option gives (List[int]): optional. defaults to [0]. which files are directly returned to the metric. if len==1, str is returned; else, list is returned. + only support for single file now: + time_suffix(bool): optional. defaults to False. if True, append the current time in required format. + time_format(str): optional. defaults to "%Y_%m_%d". format of the time suffix. """ - + time_format = "%Y_%m_%d" if not config.get("multi", False): paths: List[str] = [config["path"]] dests: List[str] = [config["dest"]] else: paths: List[str] = config["path"] dests: List[str] = config["dest"] + if "time_suffix" in config.keys() and config["time_suffix"]: + if "time_format" in config.keys(): + time_format = config["time_format"] + # Insert time before . in file type suffix + paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths] + dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests] + cache_paths: List[str] = [] gives: Set[int] = set(config.get("gives", [0])) @@ -85,7 +95,8 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option cache_paths.append(_path) with open(_path, "wb") as f: f.write(file) - + # debug + print(cache_paths) return cache_paths[0] if len(cache_paths)==1 else cache_paths diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index db04aea..3ee4695 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -195,3 +195,10 @@ def get_accessibility_tree(env, *args) -> str: accessibility_tree: str = env.controller.get_accessibility_tree() logger.debug("AT@eval: %s", accessibility_tree) return accessibility_tree + +def get_time_diff_range(env, config) -> str: + try: + return config["diff_range_in_minutes"] + except: + logger.error("diff_range_in_minutes not found in config.") + return None \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 5a7025c..92d5877 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -2,7 +2,8 @@ from .basic_os import ( check_gnome_favorite_apps, is_utc_0, check_text_enlarged, - check_moved_jpgs + check_moved_jpgs, + is_in_vm_clickboard ) from .chrome import ( is_expected_tabs, @@ -18,7 +19,8 @@ from .chrome import ( is_expected_search_query, is_expected_active_tab, is_expected_url_pattern_match, - is_added_to_steam_cart + is_added_to_steam_cart, + is_expected_installed_extensions ) from .docs import ( compare_font_names, @@ -45,7 +47,8 @@ from .docs import ( is_first_line_centered, check_file_exists, check_tabstops, - compare_contains_image + compare_contains_image, + compare_docx_files_and_ignore_new_lines ) from .general import ( check_csv, @@ -57,7 +60,15 @@ from .general import ( is_in_list, fuzzy_match, check_include_exclude, - check_direct_json_object + check_direct_json_object, + compare_time_in_speedtest_results, + is_included_all_json_objects, + is_gold_text_included_in_pdf, + check_csv_line_number, + file_contains, + compare_terminal_and_txt, + fuzzy_place_math, + compare_python_pure_text ) from .gimp import ( check_brightness_decrease_and_structure_sim, @@ -124,7 +135,7 @@ from .vscode import ( check_json_settings, check_json_keybindings ) - +from .calc import compare_conference_city_in_order def infeasible(): pass diff --git a/desktop_env/evaluators/metrics/basic_os.py b/desktop_env/evaluators/metrics/basic_os.py index e674d6d..aac1f7c 100644 --- a/desktop_env/evaluators/metrics/basic_os.py +++ b/desktop_env/evaluators/metrics/basic_os.py @@ -56,3 +56,15 @@ def check_moved_jpgs(directory_list, rule): return 1 else: return 0 + +def is_in_vm_clickboard(config, terminal_output): + print("terminal_output: ") + print(terminal_output) + print("config: ") + print(config) + expected_results = config["expected"] + # check if terminal_output has expected results + if not isinstance(expected_results, list): + return 1 if expected_results in terminal_output else 0 + else: + return 1 if all(result in terminal_output for result in expected_results) else 0 \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/calc.py b/desktop_env/evaluators/metrics/calc.py new file mode 100644 index 0000000..9f96cb4 --- /dev/null +++ b/desktop_env/evaluators/metrics/calc.py @@ -0,0 +1,24 @@ +import openpyxl + +def compare_conference_city_in_order( actual_city_list_path, expected_city): + expected_city_list = expected_city["expected"] + print(f"Reading csv file from {actual_city_list_path}") + wb = openpyxl.load_workbook(actual_city_list_path) + sheet = wb.active + actual_city_list = [] + for row in sheet["C2:C22"]: + for cell in row: + actual_city_list.append(cell.value) + # expected_city is the city that we want to compare with the actual city list + # must in order index + # debug + print("expected_city_list:") + print(expected_city_list) + print("actual_city_list_path:") + print(actual_city_list) + wrong_list = [] + for i in range(len(actual_city_list)): + if expected_city_list[i] not in actual_city_list[i]: + wrong_list.append(i) + print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + return True if len(wrong_list) == 0 else False \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index 0a76bf0..f380a8e 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -61,6 +61,12 @@ def is_expected_url_pattern_match(result, rules) -> float: return 1. +def is_expected_installed_extensions(installed_extensions, expected) -> float: + print("installed_extensions: ") + print(installed_extensions) + expected_extensions = expected["expected"] + return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed + def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float: """ Checks if the expected tabs are open in Chrome. diff --git a/desktop_env/evaluators/metrics/demo.py b/desktop_env/evaluators/metrics/demo.py new file mode 100644 index 0000000..4c5b55d --- /dev/null +++ b/desktop_env/evaluators/metrics/demo.py @@ -0,0 +1,28 @@ +import fitz # PyMuPDF + +def extract_answers_from_pdf(pdf_file): + # 打开PDF文件 + doc = fitz.open(pdf_file) + answers = [] + + # 遍历每一页 + for page in doc: + # 提取当前页的文本 + text = page.get_text() + # 分割文本为行 + lines = text.split('\n') + for line in lines: + if line.strip(): # 排除空白行 + # 分割等号，提取答案 + parts = line.split('=') + if len(parts) > 1: + answer = parts[-1].strip() # 取等号后的部分为答案 + answers.append(answer) + + return answers + +# 假设你的文件名是'math_problems.pdf' +pdf_file = '/Users/lxc/Desktop/calculus.pdf' +answers = extract_answers_from_pdf(pdf_file) +for i, answer in enumerate(answers, 1): + print(f"题目{i}的答案是: {answer}") diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index 914211a..8480631 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -96,12 +96,19 @@ def compare_docx_files(file1, file2, **options): if text1 != text2: return 0 else: + print("ignore_blanks=false") if len(doc1_paragraphs) != len(doc2_paragraphs): + print(doc1_paragraphs) + print(doc2_paragraphs) + print(len(doc1_paragraphs)) + print(len(doc2_paragraphs)) return 0 - + print("in compare") # Compare each paragraph for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): if p1 != p2: + print(p1) + print(p2) return 0 return 1 @@ -490,6 +497,39 @@ def compare_docx_lines(file1, file2): return 0 +def compare_docx_files_and_ignore_new_lines(file1, file2, **options): + ignore_blanks = options.get('ignore_blanks', True) + + # Determine file types and load documents + if file1.endswith('.docx') and file2.endswith('.docx'): + doc1 = Document(file1) + doc2 = Document(file2) + # First, delete all the blank in paragraphs + doc1 = [p for p in doc1.paragraphs if p.text != ''] + doc2 = [p for p in doc2.paragraphs if p.text != ''] + doc1_paragraphs = [p.text for p in doc1] + doc2_paragraphs = [p.text for p in doc2] + else: + # Unsupported file types or mismatch + print("Unsupported file types or mismatch between file types.") + return 0 + + # Process and compare documents + if ignore_blanks: + text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() + text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip() + if text1 != text2: + return 0 + else: + if len(doc1_paragraphs) != len(doc2_paragraphs): + return 0 + # Compare each paragraph + for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): + if p1 != p2: + return 0 + return 1 + + # Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated def compare_highlighted_text(file1, file2): def extract_highlighted_text(file_path): diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 550c54b..71ff95b 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -3,15 +3,18 @@ import functools import json import operator import re +import pdfplumber import sqlite3 from numbers import Number from typing import Callable, Any, Union from typing import Dict, List, Pattern - +import datetime +import pandas as pd import lxml.etree from lxml.cssselect import CSSSelector from lxml.etree import _Element from rapidfuzz import fuzz +from docx import Document from .utils import _match_record, _match_value_to_rule @@ -46,13 +49,27 @@ def is_in_list(result, rules) -> float: return 0. - def fuzzy_match(result, rules) -> float: expect = rules["expected"] return fuzz.ratio(result, expect) / 100. +def fuzzy_place_math(result_file_path, rules) -> float: + expect = rules["expected"] # a list of possible answers + # read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line + doc = Document(result_file_path) + words_list = [] + for para in doc.paragraphs: + words_list.extend(para.text.split()) + # 打印出提取的单词列表 + print(words_list) + for word in words_list: + if not any(ans in word for ans in expect): + print("Wrong place:", word) + return 0 + return 1 + def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ Args: @@ -227,15 +244,138 @@ def check_direct_json_object(result, rules)->float: One of the most commonly used function to evalute. Compare two json objects directly. """ + if isinstance(result, str): + # remove blanks before and after result + result = result.strip() + # replace all ' with " + result = result.replace("'", '"') + # load json object + result = json.loads(result) print("result: ") print(result) print("expected: ") print(rules["expected"]) if result is None: return 0. - expected_json = rules["expected"] - for key in expected_json.keys(): - expected_value = expected_json.get(key) - if expected_value != result.get(key): - return 0. - return 1.0 \ No newline at end of file + expect_in_result = rules.get("expect_in_result", False) + if not expect_in_result: + expected_json = rules["expected"] + for key in expected_json.keys(): + expected_value = expected_json.get(key) + if expected_value != result.get(key): + return 0. + return 1.0 + else: + expected_json = rules["expected"] + for key in expected_json.keys(): + expected_value = expected_json.get(key) + if expected_value not in result.get(key): + return 0. + return 1.0 + +def compare_time_in_speedtest_results(speedtest_result_path, time_diff): + # open the speedtest results file(csv) + date_col = None + with open(speedtest_result_path, 'r') as f: + reader = pd.read_csv(f) + for column in reader.columns: + if column.startswith('TEST_DATE'): + date_col = column + break + now_date_time = datetime.datetime.now().strftime('%H:%M') + for date in reader[date_col]: + date_time = date[-5:] + # compare the date time with the current date time, if time diff less than time_diff para, then return true + if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, '%H:%M')).total_seconds()) / 60 < int(time_diff): + return False + return True + + +def is_included_all_json_objects(gold_file_path, result_file_path): + print("gold_file_path: ") + print(gold_file_path) + print("result_file_path: ") + print(result_file_path) + # two json file, check if all the key-value pair in gold_file_path is included in result_file_path + with open(gold_file_path, 'r') as f: + gold_json = json.load(f) + with open(result_file_path, 'r') as fr: + result_json = json.load(fr) + for key in gold_json.keys(): + if key not in result_json.keys() or gold_json[key] != result_json[key]: + return False + return True + + +def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path): + print("gold_text_path: ") + print(gold_text_path) + print("pdf_file_path: ") + print(pdf_file_path) + # gold file is a json file, we need to check all the value in json are included in pdf file. + with open(gold_text_path, 'r') as f: + gold_json = json.load(f) + with pdfplumber.open(pdf_file_path) as pdf: + text = '' + for page in pdf.pages: + text += page.extract_text() + false_list = [] + for key in gold_json.keys(): + if gold_json[key] not in text: + false_list.append(key) + if len(false_list) > 0: + print("false_list: ") + print(false_list) + return False + else: + return True + + +def file_contains(file_path, config): + # file_path ends with .txt + if not file_path : + return False + with open(file_path, 'r') as f: + file_text = f.read() + for text in config["expected"]: + if text not in file_text: + return False + return True + +def check_csv_line_number(file_path, line_number): + # check file_path suffix + if not file_path.endswith('.csv'): + return False + # check line number + with open(file_path, 'r') as f: + reader = csv.reader(f) + line_count = sum(1 for row in reader) + return True if line_count == int(line_number["expected"]) else False + + +def compare_terminal_and_txt(txt_file_path, terminal_output): + # read txt file content + with open(txt_file_path, 'r') as f: + txt_file_content = f.read() + # compare terminal output with txt file content + return True if terminal_output == txt_file_content else False + + +def compare_python_pure_text(py_file_path, gold_file_path): + # first, change the suffix of gold_file from .txt to .py + print("py_file_path: ") + print(py_file_path) + print("gold_file_path: ") + print(gold_file_path) + # gold_file_path = gold_file_path.replace('.txt', '.py') + def remove_whitespace(text): + return ''.join(text.split()) + with open(py_file_path, 'r') as file1: + content1 = file1.read() + with open(gold_file_path, 'r') as file2: + content2 = file2.read() + # 移除文件内容中的所有空白字符 + content1_no_whitespace = remove_whitespace(content1) + content2_no_whitespace = remove_whitespace(content2) + # 比较处理后的文件内容 + return content1_no_whitespace == content2_no_whitespace \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/utils.py b/desktop_env/evaluators/metrics/utils.py index 6f444f8..1afc892 100644 --- a/desktop_env/evaluators/metrics/utils.py +++ b/desktop_env/evaluators/metrics/utils.py @@ -1,10 +1,12 @@ import builtins +import datetime import functools import itertools import logging import operator import re import zipfile +import pandas as pd from typing import Any, TypeVar, Union, Iterable, Optional, Callable from typing import Dict, List, Set, Match, Tuple, Pattern from urllib.parse import urlparse, urlunparse diff --git a/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json b/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json index a3630a8..2665e5b 100644 --- a/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json +++ b/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json @@ -53,17 +53,32 @@ "chrome" ], "evaluator": { - "func": "is_expected_active_tab", - "result": { + "func": ["is_expected_active_tab", "is_expected_active_tab"], + "conj": "or", + "result": [ + { "type": "active_url_from_accessTree", "goto_prefix": "https://www." - }, - "expected": { + }, + { + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." + } + ], + "expected": [ + { "type": "rule", "rules": { "type": "url", "url": "https://www.drugs.com/npc/" } - } + }, + { + "type": "rule", + "rules": { + "type": "url", + "url": "https://www.drugs.com/npp/" + } + }] } } diff --git a/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json b/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json index ef3fe12..c934549 100644 --- a/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json +++ b/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json @@ -1,7 +1,7 @@ { "id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca", "snapshot": "chrome", - "instruction": "Can you make a new folder for me on that bookmarks bar in my internet browser? Let's call it 'Favorites.'", + "instruction": "Can you make a new folder for me on the bookmarks bar in my internet browser? Let's call it 'Favorites.'", "source": "https://www.youtube.com/watch?v=IN-Eq_UripQ", "config": [ { diff --git a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json index 0b12421..147425f 100644 --- a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json +++ b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json @@ -43,19 +43,35 @@ "chrome" ], "evaluator": { - "func": "exact_match", - "result": { + "func": ["exact_match", "exact_match"], + "conj": "or", + "result": [ + { "type": "url_dashPart", "goto_prefix": "https://www.", "partIndex": -1, "needDeleteId": false, "returnType": "string" - }, - "expected": { + }, + { + "type": "url_dashPart", + "goto_prefix": "https://www.", + "partIndex": -1, + "needDeleteId": false, + "returnType": "string" + }], + "expected": [ + { "type": "rule", "rules": { "expected": "tamiflu.html#side-effects" } - } + }, + { + "type": "rule", + "rules": { + "expected": "tamiflu-side-effects.html" + } + }] } } \ No newline at end of file diff --git a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json index 460c60d..cbd3b75 100644 --- a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json +++ b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json @@ -53,7 +53,7 @@ "rules": { "expected": [ "AgeAppropriate:Kids", - "search=spider-man%20toys", + "search=spider[-%20]?man%20toys", "S=4" ] } diff --git a/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json new file mode 100644 index 0000000..4525376 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json @@ -0,0 +1,49 @@ +{ + "id": "2373b66a-092d-44cb-bfd7-82e86e7a3b4d", + "snapshot": "multiapps", + "instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor the system CPU usage, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop, and convert the txt to csv file with the same name.", + "source": "author", + "config": [ + { + "type": "command", + "parameters":{ + "command": "echo lixiaochuan20 | sudo -S apt-get update && echo lixiaochuan20 | sudo -S apt-get install sysstat", + "shell": "true" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "calc" + ], + "evaluator": { + "func": ["file_contains", "check_csv_line_number"], + "result": + [ + { + "type": "vm_file", + "path": "/home/user/Desktop/System_Resources_Report.txt", + "dest": "System_Resources_Report.txt" + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/System_Resources_Report.csv", + "dest": "System_Resources_Report.csv" + } + ], + "expected": + [ + { + "type": "rule", + "rules" :{ + "expected": ["tps", "rtps", "wtps", "dtps", "bread/s", "bwrtn/s", "bdscd/s", "Average", "Linux"] + } + }, + { + "type": "rule", + "rules": { + "expected": "33" + } + }] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json b/evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json new file mode 100644 index 0000000..20a274f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json @@ -0,0 +1,69 @@ +{ + "id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38", + "snapshot": "multiapps", + "instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed (if the dir does not exist, create it).", + "source": "https://www.speedtest.net/", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.speedtest.net/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(0.5);" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps":[ + "os", + "browser" + ], + "evaluator":{ + "func": "compare_time_in_speedtest_results", + "result":{ + "type": "vm_file", + "path": "/home/user/Test/Speed/Speedtest Results Export-.csv", + "dest": "Speedtest Results Export-.csv", + "time_suffix": true + }, + "expected":{ + "type": "time_diff_range", + "diff_range_in_minutes": "60" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json b/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json new file mode 100644 index 0000000..4bd4c25 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json @@ -0,0 +1,52 @@ +{ + "id": "3eb2a122-a5e3-4f89-9820-f7fa1a582969", + "snapshot": "multiapps", + "instruction": "Please search online for the submission deadline and venue of the ICLR main conference in 2035, and copy it to my clipboard. If not yet publicized, copy None.", + "source": "author", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "command", + "parameters":{ + "command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc", + "shell": "true" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "chrome" + ], + "evaluator": { + "func": "is_in_vm_clickboard", + "expected": { + "type": "vm_command_line", + "command": "xsel --clipboard --output", + "shell": "true" + }, + "result": { + "type": "rule", + "rules": { + "expected": ["None"] + } + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json b/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json new file mode 100644 index 0000000..dfb60ed --- /dev/null +++ b/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json @@ -0,0 +1,70 @@ +{ + "id": "5bc63fb9-276a-4439-a7c1-9dc76401737f", + "snapshot": "multiapps", + "instruction": "I have a JSON-formatted data file opened now that stores the responses of several large language models on a specific dataset. Now, I want to filter out all the responses from Gemini and specifically look at the sentences in the responses that contain \"Iliad\". Please copy all of Gemini's responses(values) from the JSON file, paste them into a Word document named \"gemini_results.docx\" under Desktop. Each response should be a new paragraph and sepreated by a new line. Highlight the all the \"Iliad\" word.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6", + "path": "/home/user/Desktop/llm_answers.json" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/llm_answers.json" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "gemini_results.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + ] + } + } + ], + "func": "compare_docx_files_and_ignore_new_lines", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1N5LlYhrRWtcC24LKiDQAYJYqCDYMBh9b", + "dest": "gemini_results_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/gemini_results.docx", + "dest": "gemini_results.docx" + }, + "options": { + "ignore_blanks": false + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json b/evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json new file mode 100644 index 0000000..9408e32 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json @@ -0,0 +1,70 @@ +{ + "id": "65e62ec6-4603-4c67-b4d2-07830deb285b", + "snapshot": "multiapps", + "instruction": "I've recently become very interested in large language models. I tried entering some questions into several large language models and stored the answers in a file named \"answers.json\" under Desktop. Now, I want to focus only on the answers from GPT-4. Please help me copy all the GPT-4 answers into a Word document named \"results.docx\" under Desktop, one answer per paragraph.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6", + "path": "/home/user/Desktop/answers.json" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/answers.json" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "results.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + ] + } + } + ], + "func": "compare_docx_files_and_ignore_new_lines", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1edtU4kHecg7oxerf1jvd1-ak-2WmEla3", + "dest": "results_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/results.docx", + "dest": "results.docx" + }, + "options": { + "ignore_blanks": false + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json new file mode 100644 index 0000000..eee89fc --- /dev/null +++ b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json @@ -0,0 +1,54 @@ +{ + "id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a", + "snapshot": "multiapps", + "instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.", + "source": "author", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1MhoRWCk-_ZZYg1agfQWHaOEYxXOrXLKC", + "path": "/home/user/Desktop/ConferenceCity.xlsx" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1p10oM6S3iZ6x2UVCgZE1mQ9D1yeZGCbO", + "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/ConferenceCity.xlsx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "calc", "chrome", "os" + ], + "evaluator": { + "func": "compare_conference_city_in_order", + "expected": { + "type": "rule", + "rules":{ + "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"] + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/ConferenceCity.xlsx", + "dest": "ConferenceCity.xlsx" + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json b/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json new file mode 100644 index 0000000..e225747 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json @@ -0,0 +1,51 @@ +{ + "id": "716a6079-22da-47f1-ba73-c9d58f986a38", + "snapshot": "multiapps", + "instruction": "I remember there is a file named \"secret.docx\" on this computer, but I can't remember where it is. Please find the path where this file is stored and copy it to the clipboard.", + "source": "", + "config": [ + { + "type": "command", + "parameters": { + "command": "mkdir -p /home/user/Data1/List1 && mkdir -p /home/user/Data2/List2 && mkdir -p /home/user/Data3/List3 && mkdir -p /home/user/Data4/List4 && mkdir -p /home/user/Data5/List5", + "shell" :"true" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1kjfHFNmAy4ZVqGgFsxPDxWwMuZkCHxAv", + "path": "/home/user/Data3/List3/secret.docx" + } + ] + } + }, + { + "type": "command", + "parameters":{ + "command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc", + "shell": "true" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "terminal" + ], + "evaluator": { + "func": "is_in_vm_clickboard", + "expected": { + "type": "vm_command_line", + "command": "xsel --clipboard --output", + "shell": "true" + }, + "result": { + "type": "rule", + "rules": { + "expected": "/home/user/Data3/List3/secret.docx" + } + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json new file mode 100644 index 0000000..c9c97fe --- /dev/null +++ b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json @@ -0,0 +1,50 @@ +{ + "id": "7ff48d5b-2df2-49da-b500-a5150ffc7f18", + "snapshot": "multiapps", + "instruction": "I am a Chinese citizen and I want to go to Macau to watch a concert recently, but I have not yet applied for a visa for Macau. I live in Futian District, Shenzhen City. I heard that Shenzhen currently has 24-hour self-service check-in machines. Please help me find the addresses of 5 24-hour self-service check-in machines in Futian District and save them in Chinese in this open word document.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1wrxlOrxZr3I5vaF1KRzc9nA9UcWW2ReU", + "path": "/home/user/Desktop/AllLocations.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/AllLocations.docx" + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "AllLocations.docx - LibreOffice Writer" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os", "writer" + ], + "evaluator": { + "func": "fuzzy_place_math", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/AllLocations.docx", + "dest": "AllLocations.docx" + }, + "expected": { + "type": "rule", + "rules":{ + "expected": ["正义", "海天综合大厦", "盛世家园", "香梅路", "金沙嘴大厦", "卓悦汇", "昌泰公寓", "天安数码城", "振兴路", "梅岗南街", "滨河大道", "上沙", "香蜜原著", "八卦岭", "竹园", "竹子林五路", "体育中心", "赤尾", "书城中心广场", "侨城东路", "沙头所", "水围村", "黄木岗", "华强广场", "国大药房","科学馆","江苏宾馆", "梅林"] + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json new file mode 100644 index 0000000..a5aa3a3 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json @@ -0,0 +1,66 @@ +{ + "id": "873cafdd-a581-47f6-8b33-b9696ddb7b05", + "snapshot": "multiapps", + "instruction": "My friend is a \"plugin guru\" and he recommended some good plug-ins to me. Please go to the Chrome plug-in store and install all the listed plug-ins.", + "source": "author", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1el4WdiJVpUP49FABN_DpC7yy7obgJSC5", + "path": "/home/user/Desktop/Recommended_plugin_list.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/Recommended_plugin_list.docx" + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Recommended_plugin_list.docx - LibreOffice Writer" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "chrome", "Writer" + ], + "evaluator": { + "func": "is_expected_installed_extensions", + "expected": { + "type": "rule", + "rules":{ + "expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"] + } + }, + "result": { + "type": "find_installed_extension_name" + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json new file mode 100644 index 0000000..17aac76 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json @@ -0,0 +1,88 @@ +{ + "id": "9f3bb592-209d-43bc-bb47-d77d9df56504", + "snapshot": "multiapps", + "instruction": "I downloaded a video to practice listening, but I don't know how to remove the subtitles. Please help me remove the subtitles from the video and export it as \"subtitles.srt\" and store it in the same directory as the video.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1YrPG1sXDACdXlrBlW8-RUzovWUJ0qoC5", + "path": "/home/user/video.txt" + } + ] + } + }, + { + "type": "execute", + "parameters":{ + "command":"echo lixiaochuan20 | sudo -S mv /home/user/video.txt /home/user/video.mp4", + "shell": "true" + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/video.mp4" + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "video.mp4 - VLC media player" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "vlc" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1Q6GV39h74bc_8_kFKjNW3RMxUSgXaaWR", + "path": "/home/user/subtitles_Gold.srt" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=14eBaLVfUekkpj3dPkdDwHehHRmzx2njy", + "path": "/home/user/Desktop/subtitles_script.txt" + } + ] + } + }, + { + "type": "command", + "parameters": { + "command": "echo lixiaochuan20 | sudo -S pip install pysrt", + "shell" :"true" + } + } + ], + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "echo lixiaochuan20 | mv /home/user/Desktop/subtitles_script.txt /home/user/Desktop/subtitles_script.py && python /home/user/Desktop/subtitles_script.py", + "shell" :"true" + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "true\n" + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json b/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json new file mode 100644 index 0000000..dafe64d --- /dev/null +++ b/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json @@ -0,0 +1,62 @@ +{ + "id": "a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a", + "snapshot": "multiapps", + "instruction": "I have developed a new Chrome extension myself, so it needs to be installed manually. Please help me install the extension located in the Desktop directory into the Chrome browser.", + "source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en", + "config": [ + { + "type": "download", + "parameters": { + "files":[ + { + "url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download", + "path":"/home/user/Desktop/helloExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S apt-get update -y && echo password | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip", + "shell": true + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os" + ], + "evaluator": { + "func": "is_in_list", + "result": { + "type": "find_unpacked_extension_path" + }, + "expected": { + "type": "rule", + "rules": { + "expected": "/home/user/Desktop/helloExtension" + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json new file mode 100644 index 0000000..0fc1380 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json @@ -0,0 +1,80 @@ +{ + "id": "ce2b64a2-ddc1-4f91-8c7d-a88be7121aac", + "snapshot": "multiapps", + "instruction": "There are several pictures of mountains in my Pictures directory, but I don’t know the names of these mountains. Please help me identify these pictures and change the names of these pictures to the names of the mountains in the pictures.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1LSFS3AI4t5aOIEUby73ZHvURqnx5cw7C", + "path": "/home/user/Pictures/picture1.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1pdRoU_FnqoDgenvp1WzFJZrsGnsexRoo", + "path": "/home/user/Pictures/picture2.jpg" + + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1ajGd9b_YvE1Ay59Pqh8RnePgbnepGc_g", + "path": "/home/user/Pictures/picture3.jpg" + + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os", "image" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1cbBn9BqtkmiZH-nNJAfQ_RYjaAUGHHTG", + "path": "/home/user/Desktop/image_script.txt" + } + ] + } + } + ], + "func": "check_direct_json_object", + "result":{ + "type": "vm_command_line", + "command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/image_script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py", + "shell": "true" + }, + "expected": { + "type": "rule", + "rules":{ + "expected":{ + "ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": "Kilimanjaro", + "999957c8de835bfa420d198270e7a6b079ee20ff53a3f214491e8134768a7c0b": "Himalayas", + "79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": "Hua" + }, + "expect_in_result": true + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json new file mode 100644 index 0000000..f7adb15 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json @@ -0,0 +1,75 @@ +{ + "id": "da922383-bfa4-4cd3-bbad-6bebab3d7742", + "snapshot": "multiapps", + "instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs you are opening now in pdf format to /home/user/Documents/Blogs.", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://lilianweng.github.io/posts/2023-06-23-agent/", + "https://lilianweng.github.io/posts/2024-02-05-human-data-quality/" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=19SJ4ydB36lwIK4K4B57sl3bsY9T-3WNv", + "path": "/home/user/Desktop/script.txt" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "pip install PyMuPDF", + "shell": "true" + } + } + ], + "func": "exact_match", + "result":{ + "type": "vm_command_line", + "command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py", + "shell": "true" + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "[1, 1]\n" + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json b/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json new file mode 100644 index 0000000..a7acd4e --- /dev/null +++ b/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json @@ -0,0 +1,53 @@ +{ + "id": "dd60633f-2c72-42ba-8547-6f2c8cb0fdb0", + "snapshot": "multiapps", + "instruction": "I ran some simple code demos on the currently open google colab, and I think the effect is pretty good. Please help me extract the code in all code boxes, merge it into a \"task.py\" file and store it in the local Home directory.", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://colab.research.google.com/github/stanfordnlp/dspy/blob/main/intro.ipynb#scrollTo=L1BHAoL_LRd7" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os", "writer" + ], + "evaluator": { + "func": "compare_python_pure_text", + "result": { + "type": "vm_file", + "path": "/home/user/colab.py", + "dest": "colab.py" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1Z1Vl7SOXf0yjoKttkZD1oYbLM7TgXr5s", + "dest": "colab_Gold.txt" + } + } + } + \ No newline at end of file From 6f0fe4f48232efab29e8c983862d074c5e2bd9e8 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Fri, 8 Mar 2024 20:39:05 +0800 Subject: [PATCH 4/9] Fix a bug in multiple apps example --- desktop_env/evaluators/metrics/chrome.py | 27 ++++++++++++++----- .../a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json | 14 +++++++--- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index fab3ebe..45b42cf 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -2,9 +2,9 @@ import logging import os import re import shutil +from itertools import product from typing import Any, Dict, List, Union -import fitz # PyMuPDF import rapidfuzz.fuzz as fuzz from bs4 import BeautifulSoup, Tag @@ -94,12 +94,24 @@ def is_expected_bookmarks(bookmarks: List[str], rule: Dict[str, Any]) -> float: elif rule['type'] == "liked_authors_websites_urls": # Check if "liked authors" folder exists liked_authors_folder = next((bookmark for bookmark in bookmarks['bookmark_bar']['children'] if - bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None) + bookmark['type'] == 'folder' and bookmark['name'] == 'Liked Authors'), None) if liked_authors_folder: # Check if it contains the specified URLs liked_authors_urls = [bookmark['url'] for bookmark in liked_authors_folder['children'] if bookmark['type'] == 'url'] - return 1. if set(liked_authors_urls) == set(rule['urls']) else 0. + + urls = rule['urls'] + + for idx, url in enumerate(urls): + if isinstance(url, str): + urls[idx] = [url] + + combinations = product(*urls) + + for combination in combinations: + if set(combination) == set(liked_authors_urls): + return 1. + return 0. else: return 0. else: @@ -140,15 +152,16 @@ def compare_pdfs(pdf1_path: Union[str, List[str]], pdf2_path: Union[str, List[st logger.info(f"[ERROR]: unexpected error occurred when comparing PDF files: {e}") return score / len(pdf2_path) + import fitz from PIL import Image -from io import BytesIO from borb.pdf import Document from borb.pdf import PDF from pathlib import Path import typing + def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float: def extract_images_from_pdf(pdf_path): pdf_document = fitz.open(pdf_path) @@ -163,14 +176,14 @@ def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float: images.append(img) return images - + def fix_pdf(in_path: Path, out_path: Path) -> None: doc: typing.Optional[Document] = None with open(in_path, "rb") as fh: doc = PDF.loads(fh) with open(out_path, "wb") as fh: PDF.dumps(fh, doc) - + fix_pdf(Path(pdf1_path), Path(pdf1_path)) fix_pdf(Path(pdf2_path), Path(pdf2_path)) @@ -183,7 +196,7 @@ def compare_pdf_images(pdf1_path: str, pdf2_path: str, **kwargs) -> float: for img1, img2 in zip(images1, images2): if img1.tobytes() != img2.tobytes(): return 0. - + return 1. diff --git a/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json b/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json index f5e637f..dbb3f00 100644 --- a/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json +++ b/evaluation_examples/examples/multi_apps/a82b78bb-7fde-4cb3-94a4-035baf10bcf0.json @@ -23,6 +23,12 @@ ] } }, + { + "type": "sleep", + "parameters": { + "seconds": 2 + } + }, { "type": "download", "parameters": { @@ -75,10 +81,10 @@ "Liked Authors" ], "urls": [ - "https://jimfan.me/", - "https://ai.stanford.edu/~dahuang/", - "https://yukezhu.me/", - "https://www.eas.caltech.edu/people/anima" + ["https://jimfan.me/", "https://research.nvidia.com/person/linxi-jim-fan"], + ["https://research.nvidia.com/person/de-an-huang", "https://ai.stanford.edu/~dahuang/"], + ["https://yukezhu.me/", "https://www.cs.utexas.edu/people/faculty-researchers/yuke-zhu", "https://experts.utexas.edu/yuke_zhu", "https://research.nvidia.com/person/yuke-zhu"], + ["http://tensorlab.cms.caltech.edu/users/anima/", "https://www.eas.caltech.edu/people/anima"] ] } } From 62b3b2390d8332832cf235882ef9257289b022c1 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Fri, 8 Mar 2024 23:09:11 +0800 Subject: [PATCH 5/9] Fix bugs from merging --- desktop_env/evaluators/metrics/__init__.py | 6 +++--- requirements.txt | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index d7a7520..6fdcf8d 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -20,7 +20,7 @@ from .chrome import ( is_expected_active_tab, is_expected_url_pattern_match, is_added_to_steam_cart, - is_expected_installed_extensions + is_expected_installed_extensions, compare_pdf_images ) from .docs import ( @@ -49,7 +49,7 @@ from .docs import ( check_file_exists, check_tabstops, compare_contains_image, - compare_docx_files_and_ignore_new_lines + compare_docx_files_and_ignore_new_lines, compare_docx_images, compare_image_text, compare_references @@ -72,7 +72,7 @@ from .general import ( file_contains, compare_terminal_and_txt, fuzzy_place_math, - compare_python_pure_text + compare_python_pure_text, diff_text_file, literal_match ) diff --git a/requirements.txt b/requirements.txt index fdcb95d..f6e2da6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,3 +46,4 @@ PyYaml mutagen pytesseract borb +pdfplumber From 4de0eff703091f9058042a5d2ce224d3d8249365 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 00:16:50 +0800 Subject: [PATCH 6/9] Add none file handling for doc --- desktop_env/evaluators/metrics/docs.py | 137 ++++++++++++++++--------- 1 file changed, 88 insertions(+), 49 deletions(-) diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index 3eceed5..706f3ba 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -3,8 +3,10 @@ import os import re import xml.etree.ElementTree as ET import zipfile +from io import BytesIO from typing import List, Dict, Any +from PIL import Image from docx import Document from docx.enum.text import WD_PARAGRAPH_ALIGNMENT, WD_TAB_ALIGNMENT from docx.shared import RGBColor @@ -23,6 +25,9 @@ def find_default_font(config_file_path, rules): default_font = None expected_font = rules["font_name"] + if not config_file_path: + return 0 + try: tree = ET.parse(config_file_path) root = tree.getroot() @@ -42,6 +47,9 @@ def find_default_font(config_file_path, rules): def contains_page_break(docx_file): + if not docx_file: + return 0 + doc = Document(docx_file) namespaces = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'} @@ -62,6 +70,9 @@ def compare_docx_files(file1, file2, **options): ignore_order = options.get('ignore_order', False) content_only = options.get('content_only', False) + if not file1 or not file2: + return 0 + def get_paragraph_texts_odt(document): paragraphs = document.getElementsByType(P) paragraph_texts = [] @@ -139,6 +150,9 @@ def compare_docx_files(file1, file2, **options): def compare_init_lines(file1, file2): + if not file1 or not file2: + return 0 + doc1 = Document(file1) doc2 = Document(file2) @@ -156,6 +170,9 @@ def compare_init_lines(file1, file2): def compare_docx_tables(docx_file1, docx_file2): + if not docx_file1 or not docx_file2: + return 0 + doc1 = Document(docx_file1) doc2 = Document(docx_file2) @@ -181,11 +198,10 @@ def compare_docx_tables(docx_file1, docx_file2): return 1 -from io import BytesIO -from PIL import Image - - def compare_docx_images(docx_file1, docx_file2): + if not docx_file1 or not docx_file2: + return 0 + doc1 = Document(docx_file1) doc2 = Document(docx_file2) @@ -211,6 +227,9 @@ import pytesseract def compare_image_text(image_path, rule): + if not image_path: + return 0 + img = Image.open(image_path) img_text = pytesseract.image_to_string(img) if rule['type'] == 'text': @@ -220,6 +239,9 @@ def compare_image_text(image_path, rule): def compare_line_spacing(docx_file1, docx_file2): + if not docx_file1 or not docx_file2: + return 0 + if not compare_docx_files(docx_file1, docx_file2): return 0 doc1 = Document(docx_file1) @@ -241,6 +263,9 @@ def compare_line_spacing(docx_file1, docx_file2): def compare_insert_equation(docx_file1, docx_file2): + if not docx_file1 or not docx_file2: + return 0 + if not compare_docx_files(docx_file1, docx_file2): return 0 @@ -256,6 +281,9 @@ def compare_insert_equation(docx_file1, docx_file2): def compare_font_names(docx_file, rules: List[Dict[str, Any]]): + if not docx_file: + return 0 + doc = Document(docx_file) expected_font = rules["font_name"] @@ -268,6 +296,9 @@ def compare_font_names(docx_file, rules: List[Dict[str, Any]]): def compare_subscript_contains(docx_file1, docx_file2): + if not docx_file1 or not docx_file2: + return 0 + doc1 = Document(docx_file1) doc2 = Document(docx_file2) @@ -280,6 +311,9 @@ def compare_subscript_contains(docx_file1, docx_file2): def has_page_numbers_in_footers(docx_file): + if not docx_file: + return 0 + doc = Document(docx_file) for section in doc.sections: @@ -294,6 +328,9 @@ def has_page_numbers_in_footers(docx_file): def is_first_line_centered(docx_file): + if not docx_file: + return 0 + doc = Document(docx_file) first_paragraph = doc.paragraphs[0] @@ -302,11 +339,16 @@ def is_first_line_centered(docx_file): def check_file_exists(directory, filename): + if not directory or not filename: + return 0 file_path = os.path.join(directory, filename) return 1 if os.path.isfile(file_path) else 0 def check_tabstops(docx_file1, docx_file2, **kwargs) -> float: + if not docx_file1 or not docx_file2: + return .0 + doc1: Document = Document(docx_file1) doc2: Document = Document(docx_file2) para1 = [p for p in doc1.paragraphs if p.text.strip()] @@ -342,6 +384,9 @@ def check_tabstops(docx_file1, docx_file2, **kwargs) -> float: def compare_contains_image(docx_file1, docx_file2): + if not docx_file1 or not docx_file2: + return 0 + doc1 = Document(docx_file1) doc2 = Document(docx_file2) @@ -354,6 +399,9 @@ def compare_contains_image(docx_file1, docx_file2): def evaluate_colored_words_in_tables(file_path1, file_path2, **kwargs): + if not file_path1 or not file_path2: + return 0 + if not compare_docx_files(file_path1, file_path2): return 0 document = Document(file_path1) @@ -388,6 +436,9 @@ def evaluate_colored_words_in_tables(file_path1, file_path2, **kwargs): def check_highlighted_words(file_path1, file_path2): + if not file_path1 or not file_path2: + return 0 + if not compare_docx_files(file_path1, file_path2): return 0 @@ -410,6 +461,9 @@ def check_highlighted_words(file_path1, file_path2): def evaluate_strike_through_last_paragraph(file_path1, file_path2): + if not file_path1 or not file_path2: + return 0 + if not compare_docx_files(file_path1, file_path2): return 0 document = Document(file_path1) @@ -426,6 +480,9 @@ def evaluate_strike_through_last_paragraph(file_path1, file_path2): def evaluate_conversion(file_path): + if not file_path: + return 0 + document = Document(file_path) for table in document.tables: @@ -445,6 +502,9 @@ def evaluate_conversion(file_path): def evaluate_spacing(file_path): + if not file_path: + return 0 + document = Document(file_path) # Check line spacing for introduction, body, and conclusion @@ -458,6 +518,9 @@ def evaluate_spacing(file_path): def check_italic_font_size_14(path1, path2): + if not path1 or not path2: + return 0 + if not compare_docx_files(path1, path2): return 0 document = Document(path1) @@ -471,6 +534,9 @@ def check_italic_font_size_14(path1, path2): def evaluate_alignment(docx_path): + if not docx_path: + return 0 + # Load the document doc = Document(docx_path) @@ -500,6 +566,9 @@ def evaluate_alignment(docx_path): def get_unique_train_ids(initial_file): # fixed standard + if not initial_file: + return set(), 0 + doc = Document(initial_file) train_ids = set() processed_lines = 0 @@ -516,6 +585,9 @@ def get_unique_train_ids(initial_file): # fixed standard def check_no_duplicates(initial_file, processed_file): + if not initial_file or not processed_file: + return 0 + # Open the document train_ids_ini, ini_lines = get_unique_train_ids(initial_file) doc_processed = Document(processed_file) @@ -543,6 +615,9 @@ def check_no_duplicates(initial_file, processed_file): def compare_docx_lines(file1, file2): + if not file1 or not file2: + return 0 + # Read the text of the document, line by line doc1 = Document(file1) doc1_lines = [p.text.strip() for p in doc1.paragraphs if p.text.strip()] @@ -562,6 +637,9 @@ def compare_docx_lines(file1, file2): def compare_docx_files_and_ignore_new_lines(file1, file2, **options): ignore_blanks = options.get('ignore_blanks', True) + if not file1 or not file2: + return 0 + # Determine file types and load documents if file1.endswith('.docx') and file2.endswith('.docx'): doc1 = Document(file1) @@ -594,6 +672,9 @@ def compare_docx_files_and_ignore_new_lines(file1, file2, **options): # Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated def compare_highlighted_text(file1, file2): + if not file1 or not file2: + return 0 + def extract_highlighted_text(file_path): highlighted_texts = [] @@ -631,6 +712,9 @@ def compare_highlighted_text(file1, file2): def compare_references(file1, file2, **options): + if not file1 or not file2: + return 0 + reference_indicator = options.get('reference_indicator', 'References') reference_base_result = options.get('reference_base_result', 0.5) @@ -675,48 +759,3 @@ def compare_references(file1, file2, **options): return (result - reference_base_result) / (1 - reference_base_result) else: return 0 - - -def compare_answer(file1, file2, **options): - """This is a specific function to compare the """ - # Determine file types and load documents - if file1.endswith('.docx') and file2.endswith('.docx'): - doc1 = Document(file1) - doc2 = Document(file2) - doc1_paragraphs = [p.text for p in doc1.paragraphs] - doc2_paragraphs = [p.text for p in doc2.paragraphs] - else: - # Unsupported file types or mismatch - print("Unsupported file types or mismatch between file types.") - return 0 - - # Find the references section in the paragraphs, find the idx of the last reference_indicator in the paragraph list - ref1_idx = doc1_paragraphs.index(reference_indicator) if reference_indicator in doc1_paragraphs else -1 - ref2_idx = doc2_paragraphs.index(reference_indicator) if reference_indicator in doc2_paragraphs else -1 - - if ref1_idx == -1 and ref2_idx == -1: - return 1 - - if ref1_idx == -1 or ref2_idx == -1: - return 0 - - # split the reference section into reference items, and remove the empty string items - ref1 = [p for p in doc1_paragraphs[ref1_idx + 1:] if p.strip()] - ref2 = [p for p in doc2_paragraphs[ref2_idx + 1:] if p.strip()] - - # Compare the references - - if len(ref1) != len(ref2): - return 0 - - total_similarity = 0 - for r1, r2 in zip(ref1, ref2): - # fuzzy match the references - similarity = fuzz.ratio(r1, r2) / 100.0 - total_similarity += similarity - - result = total_similarity / len(ref1) - if result >= reference_base_result: - return (result - reference_base_result) / (1 - reference_base_result) - else: - return 0 From 1e0a78a453fe7a573f0827ecc02b5df7283c9a40 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 00:30:28 +0800 Subject: [PATCH 7/9] Add none file handling for general --- desktop_env/evaluators/metrics/general.py | 77 ++++++++++++++--------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index c68fb90..02ad5bb 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -1,29 +1,30 @@ import csv +import datetime +import difflib import functools import json -import yaml +import logging import operator import re -import pdfplumber import sqlite3 from numbers import Number from typing import Callable, Any, Union from typing import Dict, List, Pattern -import datetime -import pandas as pd + import lxml.etree +import pandas as pd +import pdfplumber +import yaml +from docx import Document from lxml.cssselect import CSSSelector from lxml.etree import _Element from rapidfuzz import fuzz -from docx import Document -import difflib from .utils import _match_record, _match_value_to_rule -import logging - logger = logging.getLogger("desktopenv.metric.general") + def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float: if result is None: return 0. @@ -71,6 +72,7 @@ def is_in_list(result, rules) -> float: else: return 0. + def diff_text_file(result: str, expect: str) -> float: if result is None: return 0. @@ -81,6 +83,7 @@ def diff_text_file(result: str, expect: str) -> float: expected_lines: List[str] = f.read().splitlines() return difflib.SequenceMatcher(a=result_lines, b=expected_lines).ratio() + def fuzzy_match(result, rules) -> float: expect = rules["expected"] @@ -88,7 +91,7 @@ def fuzzy_match(result, rules) -> float: def fuzzy_place_math(result_file_path, rules) -> float: - expect = rules["expected"] # a list of possible answers + expect = rules["expected"] # a list of possible answers # read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line doc = Document(result_file_path) words_list = [] @@ -102,6 +105,7 @@ def fuzzy_place_math(result_file_path, rules) -> float: return 0 return 1 + def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ Args: @@ -209,10 +213,10 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float: return 0. if "text" in r: - match_func: Callable[[str], Number] = functools.partial( operator.eq if r["exact"] \ - else (lambda a, b: fuzz.ratio(a, b) / 100.) - , r["text"] - ) + match_func: Callable[[str], Number] = functools.partial(operator.eq if r["exact"] \ + else (lambda a, b: fuzz.ratio(a, b) / 100.) + , r["text"] + ) match_score: Number = 0 for elm in elements: match_score = max(match_score, match_func(elm.text or None)) @@ -285,7 +289,7 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str return float(metric) -def check_direct_json_object(result, rules)->float: +def check_direct_json_object(result, rules) -> float: """ One of the most commonly used function to evalute. Compare two json objects directly. @@ -320,7 +324,11 @@ def check_direct_json_object(result, rules)->float: return 0. return 1.0 + def compare_time_in_speedtest_results(speedtest_result_path, time_diff): + if not speedtest_result_path: + return 0 + # open the speedtest results file(csv) date_col = None with open(speedtest_result_path, 'r') as f: @@ -333,12 +341,17 @@ def compare_time_in_speedtest_results(speedtest_result_path, time_diff): for date in reader[date_col]: date_time = date[-5:] # compare the date time with the current date time, if time diff less than time_diff para, then return true - if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, '%H:%M')).total_seconds()) / 60 < int(time_diff): - return False - return True + if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, + '%H:%M')).total_seconds()) / 60 < int( + time_diff): + return 0 + return 1 def is_included_all_json_objects(gold_file_path, result_file_path): + if not gold_file_path or not result_file_path: + return 0 + print("gold_file_path: ") print(gold_file_path) print("result_file_path: ") @@ -350,8 +363,8 @@ def is_included_all_json_objects(gold_file_path, result_file_path): result_json = json.load(fr) for key in gold_json.keys(): if key not in result_json.keys() or gold_json[key] != result_json[key]: - return False - return True + return 0 + return 1 def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path): @@ -373,31 +386,32 @@ def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path): if len(false_list) > 0: print("false_list: ") print(false_list) - return False + return 0 else: - return True + return 1 def file_contains(file_path, config): # file_path ends with .txt - if not file_path : - return False + if not file_path: + return 1 with open(file_path, 'r') as f: file_text = f.read() for text in config["expected"]: if text not in file_text: - return False - return True + return 0 + return 1 + def check_csv_line_number(file_path, line_number): # check file_path suffix if not file_path.endswith('.csv'): - return False + return 0 # check line number with open(file_path, 'r') as f: reader = csv.reader(f) line_count = sum(1 for row in reader) - return True if line_count == int(line_number["expected"]) else False + return 1 if line_count == int(line_number["expected"]) else 0 def compare_terminal_and_txt(txt_file_path, terminal_output): @@ -405,7 +419,7 @@ def compare_terminal_and_txt(txt_file_path, terminal_output): with open(txt_file_path, 'r') as f: txt_file_content = f.read() # compare terminal output with txt file content - return True if terminal_output == txt_file_content else False + return 1 if terminal_output == txt_file_content else 0 def compare_python_pure_text(py_file_path, gold_file_path): @@ -414,13 +428,18 @@ def compare_python_pure_text(py_file_path, gold_file_path): print(py_file_path) print("gold_file_path: ") print(gold_file_path) + # gold_file_path = gold_file_path.replace('.txt', '.py') def remove_whitespace(text): return ''.join(text.split()) + with open(py_file_path, 'r') as file1: content1 = file1.read() with open(gold_file_path, 'r') as file2: content2 = file2.read() content1_no_whitespace = remove_whitespace(content1) content2_no_whitespace = remove_whitespace(content2) - return content1_no_whitespace == content2_no_whitespace \ No newline at end of file + if content1_no_whitespace == content2_no_whitespace: + return 1 + else: + return 0 From 6ea3dd856f47cb54d645ee816110013e008ec823 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Sat, 9 Mar 2024 14:03:26 +0800 Subject: [PATCH 8/9] fix multiapps bug : "26660ad1-6ebb-4f59-8cba-a8432dfe8d38" --- desktop_env/evaluators/getters/file.py | 17 ++++++++++++----- .../settings/googledrive/credentials.json | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 evaluation_examples/settings/googledrive/credentials.json diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index be878d6..39394d0 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -86,23 +86,29 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option if not config.get("multi", False): paths: List[str] = [config["path"]] dests: List[str] = [config["dest"]] - else: - paths: List[str] = config["path"] - dests: List[str] = config["dest"] + print(config) if "time_suffix" in config.keys() and config["time_suffix"]: if "time_format" in config.keys(): time_format = config["time_format"] # Insert time before . in file type suffix paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths] dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests] - + else: + paths: List[str] = config["path"] + dests: List[str] = config["dest"] + print(paths) + print(dests) + cache_paths: List[str] = [] gives: Set[int] = set(config.get("gives", [0])) for i, (p, d) in enumerate(zip(paths, dests)): + print("env cache_dir: ") + print(env.cache_dir) _path = os.path.join(env.cache_dir, d) - + print("_path: ") + print(_path) file = env.controller.get_file(p) if file is None: #return None @@ -116,6 +122,7 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option with open(_path, "wb") as f: f.write(file) # debug + print("cache_paths") print(cache_paths) return cache_paths[0] if len(cache_paths)==1 else cache_paths diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json new file mode 100644 index 0000000..c4b0d33 --- /dev/null +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -0,0 +1 @@ +{"access_token": "ya29.a0Ad52N3969wUkQepy6SBOSw9Gjg4-MNPfEUBD3OZpajVfs9wL4DbfImk-5XawHjBkTdCKKBqG5R9XIX6KvvUzQDfB2BwVwb0MfLfLJDLALia7MRdPn4j6GAES372u3bSqJNNPMwVZA9j-THb3o5svJiKcJgwcoFKeKC_xaCgYKAScSARISFQHGX2MioJPeGh_8OM6z1_BujwRe3Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0eVpYfdSAjvbCCgYIARAAGA4SNwF-L9IrAgL6KVceiEVTjtQdmPki2I3m8ejP3lzTLL2Wa3-rdrYfU7eYeKDVCS5KRxa_xCE_pPY", "token_expiry": "2024-03-08T17:16:15Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0Ad52N3969wUkQepy6SBOSw9Gjg4-MNPfEUBD3OZpajVfs9wL4DbfImk-5XawHjBkTdCKKBqG5R9XIX6KvvUzQDfB2BwVwb0MfLfLJDLALia7MRdPn4j6GAES372u3bSqJNNPMwVZA9j-THb3o5svJiKcJgwcoFKeKC_xaCgYKAScSARISFQHGX2MioJPeGh_8OM6z1_BujwRe3Q0171", "expires_in": 3599, "refresh_token": "1//0eVpYfdSAjvbCCgYIARAAGA4SNwF-L9IrAgL6KVceiEVTjtQdmPki2I3m8ejP3lzTLL2Wa3-rdrYfU7eYeKDVCS5KRxa_xCE_pPY", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file From 2291af394ff6945374b0691e7b4ca13073b39186 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Sat, 9 Mar 2024 18:06:48 +0800 Subject: [PATCH 9/9] update google drive file link in json --- desktop_env/evaluators/getters/chrome.py | 4 +- desktop_env/evaluators/metrics/calc.py | 11 +++-- desktop_env/evaluators/metrics/general.py | 13 +++--- .../99146c54-4f37-4ab8-9327-5f3291665e1e.json | 11 ++++- .../2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json | 2 +- .../3eb2a122-a5e3-4f89-9820-f7fa1a582969.json | 2 +- .../5bc63fb9-276a-4439-a7c1-9dc76401737f.json | 4 +- .../6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json | 42 +++++++++++++------ .../716a6079-22da-47f1-ba73-c9d58f986a38.json | 4 +- .../7ff48d5b-2df2-49da-b500-a5150ffc7f18.json | 37 +++++++++++++++- .../873cafdd-a581-47f6-8b33-b9696ddb7b05.json | 2 +- .../9f3bb592-209d-43bc-bb47-d77d9df56504.json | 21 ++++------ .../a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json | 2 +- .../ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json | 12 +++--- .../da922383-bfa4-4cd3-bbad-6bebab3d7742.json | 6 +-- .../dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json | 4 +- 16 files changed, 117 insertions(+), 60 deletions(-) diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index b809a8e..0304827 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -1075,8 +1075,8 @@ def get_data_delete_automacally(env, config: Dict[str, str]): try: content = env.controller.get_file(preference_file_path) data = json.loads(content) - data_delete_state = data["profile"]["exit_type"] - return data_delete_state + data_delete_state = data["profile"].get("default_content_setting_values", None) + return "true" if data_delete_state is not None else "false" except Exception as e: logger.error(f"Error: {e}") return "Google" diff --git a/desktop_env/evaluators/metrics/calc.py b/desktop_env/evaluators/metrics/calc.py index 9f96cb4..701be27 100644 --- a/desktop_env/evaluators/metrics/calc.py +++ b/desktop_env/evaluators/metrics/calc.py @@ -17,8 +17,11 @@ def compare_conference_city_in_order( actual_city_list_path, expected_city): print("actual_city_list_path:") print(actual_city_list) wrong_list = [] - for i in range(len(actual_city_list)): - if expected_city_list[i] not in actual_city_list[i]: - wrong_list.append(i) - print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + try: + for i in range(len(actual_city_list)): + if expected_city_list[i] not in actual_city_list[i]: + wrong_list.append(i) + print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + except: + return False return True if len(wrong_list) == 0 else False \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 02ad5bb..7ff7bf2 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -98,13 +98,16 @@ def fuzzy_place_math(result_file_path, rules) -> float: for para in doc.paragraphs: words_list.extend(para.text.split()) # Print out the list of extracted words + print("Your Answers: ") print(words_list) + fuzzy_score_list = [] for word in words_list: - if not any(ans in word for ans in expect): - print("Wrong place:", word) - return 0 - return 1 - + max_score = 0 + for ans in expect: + score = fuzz.ratio(word, ans) + max_score = max(max_score, score) + fuzzy_score_list.append(max_score) + return sum(fuzzy_score_list) / len(fuzzy_score_list) def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ diff --git a/evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json b/evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json index c4aa86e..480518e 100644 --- a/evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json +++ b/evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json @@ -1,7 +1,7 @@ { "id": "99146c54-4f37-4ab8-9327-5f3291665e1e", "snapshot": "chrome", - "instruction": "Please help me set Chrome to delete my browsing history automatically every time I close the browser.", + "instruction": "Please help me set Chrome to delete my browsing data automatically every time I close the browser.", "source": "https://www.youtube.com/watch?v=v0kxqB7Xa6I", "config": [ { @@ -29,6 +29,13 @@ "chrome" ], "evaluator": { + "postconfig":[{ + "type": "execute", + "parameters": { + "command": "pkill chrome", + "shell": "true" + } + }], "func": "exact_match", "result": { "type": "data_delete_automacally" @@ -36,7 +43,7 @@ "expected": { "type": "rule", "rules": { - "expected": "Crashed" + "expected": "true" } } } diff --git a/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json index 4525376..8cf2e38 100644 --- a/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json +++ b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json @@ -7,7 +7,7 @@ { "type": "command", "parameters":{ - "command": "echo lixiaochuan20 | sudo -S apt-get update && echo lixiaochuan20 | sudo -S apt-get install sysstat", + "command": "echo password | sudo -S apt-get update && echo password | sudo -S apt-get install sysstat", "shell": "true" } } diff --git a/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json b/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json index 4bd4c25..86a823f 100644 --- a/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json +++ b/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json @@ -26,7 +26,7 @@ { "type": "command", "parameters":{ - "command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc", + "command": "echo password | sudo -S apt install xsel && xsel -bc", "shell": "true" } } diff --git a/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json b/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json index dfb60ed..bb54b38 100644 --- a/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json +++ b/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6", + "url": "https://drive.google.com/uc?export=download&id=1OtN8poSktw5s9w7y28wr5JmXszMOpn2q", "path": "/home/user/Desktop/llm_answers.json" } ] @@ -55,7 +55,7 @@ "func": "compare_docx_files_and_ignore_new_lines", "expected": { "type": "cloud_file", - "path": "https://drive.google.com/uc?export=download&id=1N5LlYhrRWtcC24LKiDQAYJYqCDYMBh9b", + "path": "https://drive.google.com/uc?export=download&id=1gmg5mC7qfmPvspC-ICOIJJ3aRjuSvP8u", "dest": "gemini_results_Gold.docx" }, "result": { diff --git a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json index eee89fc..4792da5 100644 --- a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json +++ b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json @@ -9,23 +9,12 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1MhoRWCk-_ZZYg1agfQWHaOEYxXOrXLKC", + "url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH", "path": "/home/user/Desktop/ConferenceCity.xlsx" } ] } }, - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.google.com/uc?export=download&id=1p10oM6S3iZ6x2UVCgZE1mQ9D1yeZGCbO", - "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" - } - ] - } - }, { "type": "open", "parameters": { @@ -38,6 +27,35 @@ "calc", "chrome", "os" ], "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP", + "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" + } + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "ConferenceCity.xlsx - LibreOffice Calc" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + } + ], "func": "compare_conference_city_in_order", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json b/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json index e225747..beb3fca 100644 --- a/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json +++ b/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json @@ -16,7 +16,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1kjfHFNmAy4ZVqGgFsxPDxWwMuZkCHxAv", + "url": "https://drive.google.com/uc?export=download&id=1iaqtXHZs2sA11nmoLHPK9VXGO0_6xq2l", "path": "/home/user/Data3/List3/secret.docx" } ] @@ -25,7 +25,7 @@ { "type": "command", "parameters":{ - "command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc", + "command": "echo password | sudo -S apt install xsel && xsel -bc", "shell": "true" } } diff --git a/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json index c9c97fe..07d4c0d 100644 --- a/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json +++ b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1wrxlOrxZr3I5vaF1KRzc9nA9UcWW2ReU", + "url": "https://drive.usercontent.google.com/download?id=1H-GVXzyn30zxBKw02w5p5ZQzcwEZSlE4&export=download&authuser=0&confirm=t&uuid=da987f7a-7aa5-43d9-8426-741a97393726&at=APZUnTWyce0qNBfoOjOZ5pNRwIvX:1709971794981", "path": "/home/user/Desktop/AllLocations.docx" } ] @@ -42,7 +42,40 @@ "expected": { "type": "rule", "rules":{ - "expected": ["正义", "海天综合大厦", "盛世家园", "香梅路", "金沙嘴大厦", "卓悦汇", "昌泰公寓", "天安数码城", "振兴路", "梅岗南街", "滨河大道", "上沙", "香蜜原著", "八卦岭", "竹园", "竹子林五路", "体育中心", "赤尾", "书城中心广场", "侨城东路", "沙头所", "水围村", "黄木岗", "华强广场", "国大药房","科学馆","江苏宾馆", "梅林"] + "expected": [ + "深圳市福田区益田路5055号信息枢纽大厦西门一楼", + "深圳市福田区福华三路111号北三门会展中心警务室", + "深圳市福田区正义街1号", + "福田区莲科路18号莲花一村警务室", + "深圳市福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边", + "深圳市福田区香梅路2002-4号", + "福田区水围村龙景楼一楼", + "深圳市福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面", + "深圳市福田区福强路3028号金沙嘴大厦", + "深圳市福田区天安数码城昌泰公寓一楼", + "福田区泰然五路5号天安数码城9栋", + "深圳市福田区振兴路108号", + "深圳市福田区滨河大道2033号", + "深圳市福田区上沙四十八栋一巷11", + "深圳市福田区北环大道与香蜜湖路交汇处香蜜原著警务室", + "深圳市福田区八卦路38号八卦岭派出所", + "深圳市福田区宝能城市公馆B栋一楼竹园警务室", + "深圳市福田区竹子林五路12号", + "福田区福强路3028号金沙嘴大厦", + "福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边", + "福田区益田路5055号信息枢纽大厦西门一楼", + "福田区正义街1号", + "福田区香梅路2002-4号", + "福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面", + "福田区天安数码城昌泰公寓一楼", + "福田区振兴路108号", + "福田区滨河大道2033号", + "福田区上沙四十八栋一巷11", + "福田区北环大道与香蜜湖路交汇处香蜜原著警务室", + "福田区八卦路38号八卦岭派出所", + "福田区宝能城市公馆B栋一楼竹园警务室", + "福田区竹子林五路12号" + ] } } } diff --git a/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json index a5aa3a3..2d9f175 100644 --- a/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json +++ b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json @@ -28,7 +28,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1el4WdiJVpUP49FABN_DpC7yy7obgJSC5", + "url": "https://drive.google.com/uc?export=download&id=1EMpoFNW3reLgZE0lj6scRFmL-OPjWuzf", "path": "/home/user/Desktop/Recommended_plugin_list.docx" } ] diff --git a/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json index 17aac76..58c7ca7 100644 --- a/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json +++ b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json @@ -9,19 +9,12 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1YrPG1sXDACdXlrBlW8-RUzovWUJ0qoC5", - "path": "/home/user/video.txt" + "url": "https://drive.usercontent.google.com/download?id=17vIS65RVA7e9n5u8lLNqwjBl56MQpkk8&export=download&authuser=0&confirm=t&uuid=1c8882db-dd55-450d-a998-52c03bfaf801&at=APZUnTWPObUX-3Pqtusro4T5Ga5B:1709972109572", + "path": "/home/user/video.mp4" } ] } }, - { - "type": "execute", - "parameters":{ - "command":"echo lixiaochuan20 | sudo -S mv /home/user/video.txt /home/user/video.mp4", - "shell": "true" - } - }, { "type": "open", "parameters": { @@ -46,7 +39,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1Q6GV39h74bc_8_kFKjNW3RMxUSgXaaWR", + "url": "https://drive.usercontent.google.com/download?id=1oMCDx2hqj0xzkQP4le0Mn09y-vvCYXlt&export=download&authuser=0&confirm=t&uuid=2a2cac5c-f031-41f7-8504-4a013c02ee15&at=APZUnTUa188AtQ93qZG8HInS_o90:1709972317568", "path": "/home/user/subtitles_Gold.srt" } ] @@ -57,8 +50,8 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=14eBaLVfUekkpj3dPkdDwHehHRmzx2njy", - "path": "/home/user/Desktop/subtitles_script.txt" + "url": "https://drive.usercontent.google.com/download?id=19ooP4gPY2HYQVI9n31CtMlJ3q-1dr6cU&export=download&authuser=0&confirm=t&uuid=ba968219-3a9b-44d1-bee6-64e289ce4ffe&at=APZUnTXlL0jQBoia7KuzELsBCVxE:1709972303196", + "path": "/home/user/Desktop/subtitles_script.py" } ] } @@ -66,7 +59,7 @@ { "type": "command", "parameters": { - "command": "echo lixiaochuan20 | sudo -S pip install pysrt", + "command": "echo password | sudo -S pip install pysrt", "shell" :"true" } } @@ -74,7 +67,7 @@ "func": "exact_match", "result": { "type": "vm_command_line", - "command": "echo lixiaochuan20 | mv /home/user/Desktop/subtitles_script.txt /home/user/Desktop/subtitles_script.py && python /home/user/Desktop/subtitles_script.py", + "command": "python /home/user/Desktop/subtitles_script.py", "shell" :"true" }, "expected": { diff --git a/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json b/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json index dafe64d..f1696e4 100644 --- a/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json +++ b/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json @@ -9,7 +9,7 @@ "parameters": { "files":[ { - "url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download", + "url":"https://drive.google.com/uc?export=download&id=1fe-MuG57hf12cEi4qI4dEI3OYJOHYoY2", "path":"/home/user/Desktop/helloExtension.zip" } ] diff --git a/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json index 0fc1380..9628868 100644 --- a/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json +++ b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1LSFS3AI4t5aOIEUby73ZHvURqnx5cw7C", + "url": "https://drive.usercontent.google.com/download?id=1sKdxRx4Uxs-wxohYkmxPfxOUpw72BINV&export=download&authuser=0&confirm=t&uuid=2cd18b66-93a6-4771-a92b-a1b1f0adf12d&at=APZUnTUvT9iZ0IdRnnNf1ys4mZaE:1709972639977", "path": "/home/user/Pictures/picture1.jpg" } ] @@ -20,7 +20,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1pdRoU_FnqoDgenvp1WzFJZrsGnsexRoo", + "url": "https://drive.usercontent.google.com/download?id=1iquZNI4ktx2915srQ1MMlhcwCuXnmq5T&export=download&authuser=0&confirm=t&uuid=e0bfd86f-a0f1-4a9e-8344-0f645eb379dd&at=APZUnTWQnCd7bNtf8ZN4KfkQUaJ9:1709972645590", "path": "/home/user/Pictures/picture2.jpg" } @@ -32,7 +32,7 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1ajGd9b_YvE1Ay59Pqh8RnePgbnepGc_g", + "url": "https://drive.usercontent.google.com/download?id=1ie3rhHzsW293xhnmgeZm6wQj_uaFmPS7&export=download&authuser=0&confirm=t&uuid=633c8b0c-0d3f-4f07-a772-945148956244&at=APZUnTX6OIN5YfudwPVT3Mxntd9E:1709972650733", "path": "/home/user/Pictures/picture3.jpg" } @@ -51,8 +51,8 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=1cbBn9BqtkmiZH-nNJAfQ_RYjaAUGHHTG", - "path": "/home/user/Desktop/image_script.txt" + "url": "https://drive.usercontent.google.com/download?id=1eWGQ3z-Ps5p784NZjj9KyhHfX7NbHgWd&export=download&authuser=0&confirm=t&uuid=8ba167f4-8669-477b-89ee-a8f6996f43d7&at=APZUnTWShYYueBb8hRnrjjabU1ZD:1709972787510", + "path": "/home/user/Desktop/image_script.py" } ] } @@ -61,7 +61,7 @@ "func": "check_direct_json_object", "result":{ "type": "vm_command_line", - "command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/image_script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py", + "command":"python /home/user/Desktop/image_script.py", "shell": "true" }, "expected": { diff --git a/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json index f7adb15..c751d7b 100644 --- a/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json +++ b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json @@ -44,8 +44,8 @@ "parameters": { "files": [ { - "url": "https://drive.google.com/uc?export=download&id=19SJ4ydB36lwIK4K4B57sl3bsY9T-3WNv", - "path": "/home/user/Desktop/script.txt" + "url": "https://drive.usercontent.google.com/download?id=1fm4wqp91_3yzVSlenc9k2Tlh3-7QuRvu&export=download&authuser=0&confirm=t&uuid=5bb46f9e-ef04-4b52-bceb-a88f0058f696&at=APZUnTU6aWNzsiqLwAlyr_8pxKUd:1709971656456", + "path": "/home/user/Desktop/script.py" } ] } @@ -61,7 +61,7 @@ "func": "exact_match", "result":{ "type": "vm_command_line", - "command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py", + "command":"python /home/user/Desktop/script.py", "shell": "true" }, "expected": { diff --git a/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json b/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json index a7acd4e..3f801d7 100644 --- a/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json +++ b/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json @@ -45,8 +45,8 @@ }, "expected": { "type": "cloud_file", - "path": "https://drive.google.com/uc?export=download&id=1Z1Vl7SOXf0yjoKttkZD1oYbLM7TgXr5s", - "dest": "colab_Gold.txt" + "path": "https://drive.usercontent.google.com/download?id=1r0VyQZf5Hi8n8R9w9iGKMsqQaxAjYN7O&export=download&authuser=0&confirm=t&uuid=56efd2ff-4494-4026-9e96-82783b5ed600&at=APZUnTUL7BOMtV3FIoc0kXe18LWz:1709972550788", + "dest": "colab_Gold.py" } } }