From 62fd8feebb41979c358bf90705457e602badae6e Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Fri, 8 Mar 2024 19:24:15 +0800 Subject: [PATCH] xiaochuan's multiapp examples --- desktop_env/evaluators/getters/__init__.py | 6 +- desktop_env/evaluators/getters/calc.py | 15 ++ desktop_env/evaluators/getters/chrome.py | 37 +++++ desktop_env/evaluators/getters/file.py | 17 +- desktop_env/evaluators/getters/misc.py | 7 + desktop_env/evaluators/metrics/__init__.py | 21 ++- desktop_env/evaluators/metrics/basic_os.py | 12 ++ desktop_env/evaluators/metrics/calc.py | 24 +++ desktop_env/evaluators/metrics/chrome.py | 6 + desktop_env/evaluators/metrics/demo.py | 28 ++++ desktop_env/evaluators/metrics/docs.py | 42 ++++- desktop_env/evaluators/metrics/general.py | 156 +++++++++++++++++- desktop_env/evaluators/metrics/utils.py | 2 + .../0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json | 25 ++- .../2ad9387a-65d8-4e33-ad5b-7580065a27ca.json | 2 +- .../b070486d-e161-459b-aa2b-ef442d973b92.json | 26 ++- .../cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json | 2 +- .../2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json | 49 ++++++ .../26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json | 69 ++++++++ .../3eb2a122-a5e3-4f89-9820-f7fa1a582969.json | 52 ++++++ .../5bc63fb9-276a-4439-a7c1-9dc76401737f.json | 70 ++++++++ .../65e62ec6-4603-4c67-b4d2-07830deb285b.json | 70 ++++++++ .../6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json | 54 ++++++ .../716a6079-22da-47f1-ba73-c9d58f986a38.json | 51 ++++++ .../7ff48d5b-2df2-49da-b500-a5150ffc7f18.json | 50 ++++++ .../873cafdd-a581-47f6-8b33-b9696ddb7b05.json | 66 ++++++++ .../9f3bb592-209d-43bc-bb47-d77d9df56504.json | 88 ++++++++++ .../a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json | 62 +++++++ .../ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json | 80 +++++++++ .../da922383-bfa4-4cd3-bbad-6bebab3d7742.json | 75 +++++++++ .../dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json | 53 ++++++ 31 files changed, 1286 insertions(+), 31 deletions(-) create mode 100644 desktop_env/evaluators/getters/calc.py create mode 100644 desktop_env/evaluators/metrics/calc.py create mode 100644 desktop_env/evaluators/metrics/demo.py create mode 100644 evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json create mode 100644 evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json create mode 100644 evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json create mode 100644 evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json create mode 100644 evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json create mode 100644 evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json create mode 100644 evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json create mode 100644 evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json create mode 100644 evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json create mode 100644 evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json create mode 100644 evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json create mode 100644 evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json create mode 100644 evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json create mode 100644 evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 43f4c1b..d78212b 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -23,14 +23,16 @@ from .chrome import ( get_active_tab_url_parse, get_gotoRecreationPage_and_get_html_content, get_url_dashPart, - get_active_url_from_accessTree + get_active_url_from_accessTree, + get_find_installed_extension_name ) from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line, get_vm_terminal_output from .gimp import get_gimp_config_file from .impress import get_audio_in_slide from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory -from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime +from .misc import get_rule, get_accessibility_tree, get_rule_relativeTime, get_time_diff_range from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player from .vscode import get_vscode_config +from .calc import get_conference_city_in_order \ No newline at end of file diff --git a/desktop_env/evaluators/getters/calc.py b/desktop_env/evaluators/getters/calc.py new file mode 100644 index 0000000..7985231 --- /dev/null +++ b/desktop_env/evaluators/getters/calc.py @@ -0,0 +1,15 @@ +import csv + +# I want to write a function, reads a csv file, and get all the contents in the third column in the order of rows +def get_conference_city_in_order(env, config): + # read the csv file + csv_path = config['csv_path'] + print(f"Reading csv file from {csv_path}") + with open(csv_path, 'r') as f: + reader = csv.reader(f) + # skip the header row + next(reader) + # get the third column in the order of rows + conference_city_list = [row[2] for row in reader] + return conference_city_list + \ No newline at end of file diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 48060a6..962d8fe 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -919,6 +919,43 @@ def get_find_unpacked_extension_path(env, config: Dict[str, str]): return "Google" +def get_find_installed_extension_name(env, config: Dict[str, str]): + os_type = env.vm_platform + if os_type == 'Windows': + preference_file_path = env.controller.execute_python_command("""import os; print(os.path.join(os.getenv('LOCALAPPDATA'), + 'Google\\Chrome\\User Data\\Default\\Preferences'))""")['output'].strip() + elif os_type == 'Darwin': + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'Library/Application Support/Google/Chrome/Default/Preferences'))")[ + 'output'].strip() + elif os_type == 'Linux': + if "arm" in platform.machine(): + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), 'snap/chromium/common/chromium/Default/Preferences'))")[ + 'output'].strip() + else: + preference_file_path = env.controller.execute_python_command( + "import os; print(os.path.join(os.getenv('HOME'), '.config/google-chrome/Default/Preferences'))")[ + 'output'].strip() + + else: + raise Exception('Unsupported operating system') + + try: + content = env.controller.get_file(preference_file_path) + data = json.loads(content) + # Preferences store all the path of installed extensions, return them all and let metrics try to find one matches the targeted extension path + all_extensions_name = [] + all_extensions = data.get('extensions', {}).get('settings', {}) + for id in all_extensions.keys(): + name = all_extensions[id]["manifest"]["name"] + all_extensions_name.append(name) + return all_extensions_name + except Exception as e: + logger.error(f"Error: {e}") + return "Google" + + def get_data_delete_automacally(env, config: Dict[str, str]): """ This function is used to open th "auto-delete" mode of chromium diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 95c0a18..72c9c54 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,7 +1,7 @@ import os from typing import Dict, List, Set from typing import Optional, Any, Union - +from datetime import datetime import requests @@ -58,14 +58,24 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option gives (List[int]): optional. defaults to [0]. which files are directly returned to the metric. if len==1, str is returned; else, list is returned. + only support for single file now: + time_suffix(bool): optional. defaults to False. if True, append the current time in required format. + time_format(str): optional. defaults to "%Y_%m_%d". format of the time suffix. """ - + time_format = "%Y_%m_%d" if not config.get("multi", False): paths: List[str] = [config["path"]] dests: List[str] = [config["dest"]] else: paths: List[str] = config["path"] dests: List[str] = config["dest"] + if "time_suffix" in config.keys() and config["time_suffix"]: + if "time_format" in config.keys(): + time_format = config["time_format"] + # Insert time before . in file type suffix + paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths] + dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests] + cache_paths: List[str] = [] gives: Set[int] = set(config.get("gives", [0])) @@ -85,7 +95,8 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option cache_paths.append(_path) with open(_path, "wb") as f: f.write(file) - + # debug + print(cache_paths) return cache_paths[0] if len(cache_paths)==1 else cache_paths diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index db04aea..3ee4695 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -195,3 +195,10 @@ def get_accessibility_tree(env, *args) -> str: accessibility_tree: str = env.controller.get_accessibility_tree() logger.debug("AT@eval: %s", accessibility_tree) return accessibility_tree + +def get_time_diff_range(env, config) -> str: + try: + return config["diff_range_in_minutes"] + except: + logger.error("diff_range_in_minutes not found in config.") + return None \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 5a7025c..92d5877 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -2,7 +2,8 @@ from .basic_os import ( check_gnome_favorite_apps, is_utc_0, check_text_enlarged, - check_moved_jpgs + check_moved_jpgs, + is_in_vm_clickboard ) from .chrome import ( is_expected_tabs, @@ -18,7 +19,8 @@ from .chrome import ( is_expected_search_query, is_expected_active_tab, is_expected_url_pattern_match, - is_added_to_steam_cart + is_added_to_steam_cart, + is_expected_installed_extensions ) from .docs import ( compare_font_names, @@ -45,7 +47,8 @@ from .docs import ( is_first_line_centered, check_file_exists, check_tabstops, - compare_contains_image + compare_contains_image, + compare_docx_files_and_ignore_new_lines ) from .general import ( check_csv, @@ -57,7 +60,15 @@ from .general import ( is_in_list, fuzzy_match, check_include_exclude, - check_direct_json_object + check_direct_json_object, + compare_time_in_speedtest_results, + is_included_all_json_objects, + is_gold_text_included_in_pdf, + check_csv_line_number, + file_contains, + compare_terminal_and_txt, + fuzzy_place_math, + compare_python_pure_text ) from .gimp import ( check_brightness_decrease_and_structure_sim, @@ -124,7 +135,7 @@ from .vscode import ( check_json_settings, check_json_keybindings ) - +from .calc import compare_conference_city_in_order def infeasible(): pass diff --git a/desktop_env/evaluators/metrics/basic_os.py b/desktop_env/evaluators/metrics/basic_os.py index e674d6d..aac1f7c 100644 --- a/desktop_env/evaluators/metrics/basic_os.py +++ b/desktop_env/evaluators/metrics/basic_os.py @@ -56,3 +56,15 @@ def check_moved_jpgs(directory_list, rule): return 1 else: return 0 + +def is_in_vm_clickboard(config, terminal_output): + print("terminal_output: ") + print(terminal_output) + print("config: ") + print(config) + expected_results = config["expected"] + # check if terminal_output has expected results + if not isinstance(expected_results, list): + return 1 if expected_results in terminal_output else 0 + else: + return 1 if all(result in terminal_output for result in expected_results) else 0 \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/calc.py b/desktop_env/evaluators/metrics/calc.py new file mode 100644 index 0000000..9f96cb4 --- /dev/null +++ b/desktop_env/evaluators/metrics/calc.py @@ -0,0 +1,24 @@ +import openpyxl + +def compare_conference_city_in_order( actual_city_list_path, expected_city): + expected_city_list = expected_city["expected"] + print(f"Reading csv file from {actual_city_list_path}") + wb = openpyxl.load_workbook(actual_city_list_path) + sheet = wb.active + actual_city_list = [] + for row in sheet["C2:C22"]: + for cell in row: + actual_city_list.append(cell.value) + # expected_city is the city that we want to compare with the actual city list + # must in order index + # debug + print("expected_city_list:") + print(expected_city_list) + print("actual_city_list_path:") + print(actual_city_list) + wrong_list = [] + for i in range(len(actual_city_list)): + if expected_city_list[i] not in actual_city_list[i]: + wrong_list.append(i) + print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + return True if len(wrong_list) == 0 else False \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index 0a76bf0..f380a8e 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -61,6 +61,12 @@ def is_expected_url_pattern_match(result, rules) -> float: return 1. +def is_expected_installed_extensions(installed_extensions, expected) -> float: + print("installed_extensions: ") + print(installed_extensions) + expected_extensions = expected["expected"] + return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed + def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float: """ Checks if the expected tabs are open in Chrome. diff --git a/desktop_env/evaluators/metrics/demo.py b/desktop_env/evaluators/metrics/demo.py new file mode 100644 index 0000000..4c5b55d --- /dev/null +++ b/desktop_env/evaluators/metrics/demo.py @@ -0,0 +1,28 @@ +import fitz # PyMuPDF + +def extract_answers_from_pdf(pdf_file): + # 打开PDF文件 + doc = fitz.open(pdf_file) + answers = [] + + # 遍历每一页 + for page in doc: + # 提取当前页的文本 + text = page.get_text() + # 分割文本为行 + lines = text.split('\n') + for line in lines: + if line.strip(): # 排除空白行 + # 分割等号，提取答案 + parts = line.split('=') + if len(parts) > 1: + answer = parts[-1].strip() # 取等号后的部分为答案 + answers.append(answer) + + return answers + +# 假设你的文件名是'math_problems.pdf' +pdf_file = '/Users/lxc/Desktop/calculus.pdf' +answers = extract_answers_from_pdf(pdf_file) +for i, answer in enumerate(answers, 1): + print(f"题目{i}的答案是: {answer}") diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index 914211a..8480631 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -96,12 +96,19 @@ def compare_docx_files(file1, file2, **options): if text1 != text2: return 0 else: + print("ignore_blanks=false") if len(doc1_paragraphs) != len(doc2_paragraphs): + print(doc1_paragraphs) + print(doc2_paragraphs) + print(len(doc1_paragraphs)) + print(len(doc2_paragraphs)) return 0 - + print("in compare") # Compare each paragraph for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): if p1 != p2: + print(p1) + print(p2) return 0 return 1 @@ -490,6 +497,39 @@ def compare_docx_lines(file1, file2): return 0 +def compare_docx_files_and_ignore_new_lines(file1, file2, **options): + ignore_blanks = options.get('ignore_blanks', True) + + # Determine file types and load documents + if file1.endswith('.docx') and file2.endswith('.docx'): + doc1 = Document(file1) + doc2 = Document(file2) + # First, delete all the blank in paragraphs + doc1 = [p for p in doc1.paragraphs if p.text != ''] + doc2 = [p for p in doc2.paragraphs if p.text != ''] + doc1_paragraphs = [p.text for p in doc1] + doc2_paragraphs = [p.text for p in doc2] + else: + # Unsupported file types or mismatch + print("Unsupported file types or mismatch between file types.") + return 0 + + # Process and compare documents + if ignore_blanks: + text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() + text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip() + if text1 != text2: + return 0 + else: + if len(doc1_paragraphs) != len(doc2_paragraphs): + return 0 + # Compare each paragraph + for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): + if p1 != p2: + return 0 + return 1 + + # Docx file saved in the ubuntu cannot use this function to compare highlight, don't know why, deprecated def compare_highlighted_text(file1, file2): def extract_highlighted_text(file_path): diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 550c54b..71ff95b 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -3,15 +3,18 @@ import functools import json import operator import re +import pdfplumber import sqlite3 from numbers import Number from typing import Callable, Any, Union from typing import Dict, List, Pattern - +import datetime +import pandas as pd import lxml.etree from lxml.cssselect import CSSSelector from lxml.etree import _Element from rapidfuzz import fuzz +from docx import Document from .utils import _match_record, _match_value_to_rule @@ -46,13 +49,27 @@ def is_in_list(result, rules) -> float: return 0. - def fuzzy_match(result, rules) -> float: expect = rules["expected"] return fuzz.ratio(result, expect) / 100. +def fuzzy_place_math(result_file_path, rules) -> float: + expect = rules["expected"] # a list of possible answers + # read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line + doc = Document(result_file_path) + words_list = [] + for para in doc.paragraphs: + words_list.extend(para.text.split()) + # 打印出提取的单词列表 + print(words_list) + for word in words_list: + if not any(ans in word for ans in expect): + print("Wrong place:", word) + return 0 + return 1 + def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ Args: @@ -227,15 +244,138 @@ def check_direct_json_object(result, rules)->float: One of the most commonly used function to evalute. Compare two json objects directly. """ + if isinstance(result, str): + # remove blanks before and after result + result = result.strip() + # replace all ' with " + result = result.replace("'", '"') + # load json object + result = json.loads(result) print("result: ") print(result) print("expected: ") print(rules["expected"]) if result is None: return 0. - expected_json = rules["expected"] - for key in expected_json.keys(): - expected_value = expected_json.get(key) - if expected_value != result.get(key): - return 0. - return 1.0 \ No newline at end of file + expect_in_result = rules.get("expect_in_result", False) + if not expect_in_result: + expected_json = rules["expected"] + for key in expected_json.keys(): + expected_value = expected_json.get(key) + if expected_value != result.get(key): + return 0. + return 1.0 + else: + expected_json = rules["expected"] + for key in expected_json.keys(): + expected_value = expected_json.get(key) + if expected_value not in result.get(key): + return 0. + return 1.0 + +def compare_time_in_speedtest_results(speedtest_result_path, time_diff): + # open the speedtest results file(csv) + date_col = None + with open(speedtest_result_path, 'r') as f: + reader = pd.read_csv(f) + for column in reader.columns: + if column.startswith('TEST_DATE'): + date_col = column + break + now_date_time = datetime.datetime.now().strftime('%H:%M') + for date in reader[date_col]: + date_time = date[-5:] + # compare the date time with the current date time, if time diff less than time_diff para, then return true + if not abs((datetime.datetime.strptime(date_time, '%H:%M') - datetime.datetime.strptime(now_date_time, '%H:%M')).total_seconds()) / 60 < int(time_diff): + return False + return True + + +def is_included_all_json_objects(gold_file_path, result_file_path): + print("gold_file_path: ") + print(gold_file_path) + print("result_file_path: ") + print(result_file_path) + # two json file, check if all the key-value pair in gold_file_path is included in result_file_path + with open(gold_file_path, 'r') as f: + gold_json = json.load(f) + with open(result_file_path, 'r') as fr: + result_json = json.load(fr) + for key in gold_json.keys(): + if key not in result_json.keys() or gold_json[key] != result_json[key]: + return False + return True + + +def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path): + print("gold_text_path: ") + print(gold_text_path) + print("pdf_file_path: ") + print(pdf_file_path) + # gold file is a json file, we need to check all the value in json are included in pdf file. + with open(gold_text_path, 'r') as f: + gold_json = json.load(f) + with pdfplumber.open(pdf_file_path) as pdf: + text = '' + for page in pdf.pages: + text += page.extract_text() + false_list = [] + for key in gold_json.keys(): + if gold_json[key] not in text: + false_list.append(key) + if len(false_list) > 0: + print("false_list: ") + print(false_list) + return False + else: + return True + + +def file_contains(file_path, config): + # file_path ends with .txt + if not file_path : + return False + with open(file_path, 'r') as f: + file_text = f.read() + for text in config["expected"]: + if text not in file_text: + return False + return True + +def check_csv_line_number(file_path, line_number): + # check file_path suffix + if not file_path.endswith('.csv'): + return False + # check line number + with open(file_path, 'r') as f: + reader = csv.reader(f) + line_count = sum(1 for row in reader) + return True if line_count == int(line_number["expected"]) else False + + +def compare_terminal_and_txt(txt_file_path, terminal_output): + # read txt file content + with open(txt_file_path, 'r') as f: + txt_file_content = f.read() + # compare terminal output with txt file content + return True if terminal_output == txt_file_content else False + + +def compare_python_pure_text(py_file_path, gold_file_path): + # first, change the suffix of gold_file from .txt to .py + print("py_file_path: ") + print(py_file_path) + print("gold_file_path: ") + print(gold_file_path) + # gold_file_path = gold_file_path.replace('.txt', '.py') + def remove_whitespace(text): + return ''.join(text.split()) + with open(py_file_path, 'r') as file1: + content1 = file1.read() + with open(gold_file_path, 'r') as file2: + content2 = file2.read() + # 移除文件内容中的所有空白字符 + content1_no_whitespace = remove_whitespace(content1) + content2_no_whitespace = remove_whitespace(content2) + # 比较处理后的文件内容 + return content1_no_whitespace == content2_no_whitespace \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/utils.py b/desktop_env/evaluators/metrics/utils.py index 6f444f8..1afc892 100644 --- a/desktop_env/evaluators/metrics/utils.py +++ b/desktop_env/evaluators/metrics/utils.py @@ -1,10 +1,12 @@ import builtins +import datetime import functools import itertools import logging import operator import re import zipfile +import pandas as pd from typing import Any, TypeVar, Union, Iterable, Optional, Callable from typing import Dict, List, Set, Match, Tuple, Pattern from urllib.parse import urlparse, urlunparse diff --git a/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json b/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json index a3630a8..2665e5b 100644 --- a/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json +++ b/evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json @@ -53,17 +53,32 @@ "chrome" ], "evaluator": { - "func": "is_expected_active_tab", - "result": { + "func": ["is_expected_active_tab", "is_expected_active_tab"], + "conj": "or", + "result": [ + { "type": "active_url_from_accessTree", "goto_prefix": "https://www." - }, - "expected": { + }, + { + "type": "active_url_from_accessTree", + "goto_prefix": "https://www." + } + ], + "expected": [ + { "type": "rule", "rules": { "type": "url", "url": "https://www.drugs.com/npc/" } - } + }, + { + "type": "rule", + "rules": { + "type": "url", + "url": "https://www.drugs.com/npp/" + } + }] } } diff --git a/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json b/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json index ef3fe12..c934549 100644 --- a/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json +++ b/evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json @@ -1,7 +1,7 @@ { "id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca", "snapshot": "chrome", - "instruction": "Can you make a new folder for me on that bookmarks bar in my internet browser? Let's call it 'Favorites.'", + "instruction": "Can you make a new folder for me on the bookmarks bar in my internet browser? Let's call it 'Favorites.'", "source": "https://www.youtube.com/watch?v=IN-Eq_UripQ", "config": [ { diff --git a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json index 0b12421..147425f 100644 --- a/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json +++ b/evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json @@ -43,19 +43,35 @@ "chrome" ], "evaluator": { - "func": "exact_match", - "result": { + "func": ["exact_match", "exact_match"], + "conj": "or", + "result": [ + { "type": "url_dashPart", "goto_prefix": "https://www.", "partIndex": -1, "needDeleteId": false, "returnType": "string" - }, - "expected": { + }, + { + "type": "url_dashPart", + "goto_prefix": "https://www.", + "partIndex": -1, + "needDeleteId": false, + "returnType": "string" + }], + "expected": [ + { "type": "rule", "rules": { "expected": "tamiflu.html#side-effects" } - } + }, + { + "type": "rule", + "rules": { + "expected": "tamiflu-side-effects.html" + } + }] } } \ No newline at end of file diff --git a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json index 460c60d..cbd3b75 100644 --- a/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json +++ b/evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json @@ -53,7 +53,7 @@ "rules": { "expected": [ "AgeAppropriate:Kids", - "search=spider-man%20toys", + "search=spider[-%20]?man%20toys", "S=4" ] } diff --git a/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json new file mode 100644 index 0000000..4525376 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json @@ -0,0 +1,49 @@ +{ + "id": "2373b66a-092d-44cb-bfd7-82e86e7a3b4d", + "snapshot": "multiapps", + "instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor the system CPU usage, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop, and convert the txt to csv file with the same name.", + "source": "author", + "config": [ + { + "type": "command", + "parameters":{ + "command": "echo lixiaochuan20 | sudo -S apt-get update && echo lixiaochuan20 | sudo -S apt-get install sysstat", + "shell": "true" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "calc" + ], + "evaluator": { + "func": ["file_contains", "check_csv_line_number"], + "result": + [ + { + "type": "vm_file", + "path": "/home/user/Desktop/System_Resources_Report.txt", + "dest": "System_Resources_Report.txt" + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/System_Resources_Report.csv", + "dest": "System_Resources_Report.csv" + } + ], + "expected": + [ + { + "type": "rule", + "rules" :{ + "expected": ["tps", "rtps", "wtps", "dtps", "bread/s", "bwrtn/s", "bdscd/s", "Average", "Linux"] + } + }, + { + "type": "rule", + "rules": { + "expected": "33" + } + }] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json b/evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json new file mode 100644 index 0000000..20a274f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/26660ad1-6ebb-4f59-8cba-a8432dfe8d38.json @@ -0,0 +1,69 @@ +{ + "id": "26660ad1-6ebb-4f59-8cba-a8432dfe8d38", + "snapshot": "multiapps", + "instruction": "I want to test the quality of the network environment my laptop is currently in. Please measure my network situation through speedtest.net, export the measurement results, and save them to ~/Test/Speed (if the dir does not exist, create it).", + "source": "https://www.speedtest.net/", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.speedtest.net/" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Google Chrome" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(0.5);" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps":[ + "os", + "browser" + ], + "evaluator":{ + "func": "compare_time_in_speedtest_results", + "result":{ + "type": "vm_file", + "path": "/home/user/Test/Speed/Speedtest Results Export-.csv", + "dest": "Speedtest Results Export-.csv", + "time_suffix": true + }, + "expected":{ + "type": "time_diff_range", + "diff_range_in_minutes": "60" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json b/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json new file mode 100644 index 0000000..4bd4c25 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json @@ -0,0 +1,52 @@ +{ + "id": "3eb2a122-a5e3-4f89-9820-f7fa1a582969", + "snapshot": "multiapps", + "instruction": "Please search online for the submission deadline and venue of the ICLR main conference in 2035, and copy it to my clipboard. If not yet publicized, copy None.", + "source": "author", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "command", + "parameters":{ + "command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc", + "shell": "true" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "chrome" + ], + "evaluator": { + "func": "is_in_vm_clickboard", + "expected": { + "type": "vm_command_line", + "command": "xsel --clipboard --output", + "shell": "true" + }, + "result": { + "type": "rule", + "rules": { + "expected": ["None"] + } + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json b/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json new file mode 100644 index 0000000..dfb60ed --- /dev/null +++ b/evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json @@ -0,0 +1,70 @@ +{ + "id": "5bc63fb9-276a-4439-a7c1-9dc76401737f", + "snapshot": "multiapps", + "instruction": "I have a JSON-formatted data file opened now that stores the responses of several large language models on a specific dataset. Now, I want to filter out all the responses from Gemini and specifically look at the sentences in the responses that contain \"Iliad\". Please copy all of Gemini's responses(values) from the JSON file, paste them into a Word document named \"gemini_results.docx\" under Desktop. Each response should be a new paragraph and sepreated by a new line. Highlight the all the \"Iliad\" word.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6", + "path": "/home/user/Desktop/llm_answers.json" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/llm_answers.json" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "gemini_results.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + ] + } + } + ], + "func": "compare_docx_files_and_ignore_new_lines", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1N5LlYhrRWtcC24LKiDQAYJYqCDYMBh9b", + "dest": "gemini_results_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/gemini_results.docx", + "dest": "gemini_results.docx" + }, + "options": { + "ignore_blanks": false + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json b/evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json new file mode 100644 index 0000000..9408e32 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/65e62ec6-4603-4c67-b4d2-07830deb285b.json @@ -0,0 +1,70 @@ +{ + "id": "65e62ec6-4603-4c67-b4d2-07830deb285b", + "snapshot": "multiapps", + "instruction": "I've recently become very interested in large language models. I tried entering some questions into several large language models and stored the answers in a file named \"answers.json\" under Desktop. Now, I want to focus only on the answers from GPT-4. Please help me copy all the GPT-4 answers into a Word document named \"results.docx\" under Desktop, one answer per paragraph.", + "source": "", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6", + "path": "/home/user/Desktop/answers.json" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/answers.json" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "results.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + ] + } + } + ], + "func": "compare_docx_files_and_ignore_new_lines", + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1edtU4kHecg7oxerf1jvd1-ak-2WmEla3", + "dest": "results_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/results.docx", + "dest": "results.docx" + }, + "options": { + "ignore_blanks": false + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json new file mode 100644 index 0000000..eee89fc --- /dev/null +++ b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json @@ -0,0 +1,54 @@ +{ + "id": "6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a", + "snapshot": "multiapps", + "instruction": "I now want to count the meeting cities of the three machine learning conferences in the past ten years from 2013 to 2019(including 2013 and 2019). I have listed the names and years of the conferences in excel. Please fill in the vacant locations.", + "source": "author", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1MhoRWCk-_ZZYg1agfQWHaOEYxXOrXLKC", + "path": "/home/user/Desktop/ConferenceCity.xlsx" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1p10oM6S3iZ6x2UVCgZE1mQ9D1yeZGCbO", + "path": "/home/user/Desktop/ConferenceCity_Gold.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/ConferenceCity.xlsx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "calc", "chrome", "os" + ], + "evaluator": { + "func": "compare_conference_city_in_order", + "expected": { + "type": "rule", + "rules":{ + "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"] + } + }, + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/ConferenceCity.xlsx", + "dest": "ConferenceCity.xlsx" + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json b/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json new file mode 100644 index 0000000..e225747 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json @@ -0,0 +1,51 @@ +{ + "id": "716a6079-22da-47f1-ba73-c9d58f986a38", + "snapshot": "multiapps", + "instruction": "I remember there is a file named \"secret.docx\" on this computer, but I can't remember where it is. Please find the path where this file is stored and copy it to the clipboard.", + "source": "", + "config": [ + { + "type": "command", + "parameters": { + "command": "mkdir -p /home/user/Data1/List1 && mkdir -p /home/user/Data2/List2 && mkdir -p /home/user/Data3/List3 && mkdir -p /home/user/Data4/List4 && mkdir -p /home/user/Data5/List5", + "shell" :"true" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1kjfHFNmAy4ZVqGgFsxPDxWwMuZkCHxAv", + "path": "/home/user/Data3/List3/secret.docx" + } + ] + } + }, + { + "type": "command", + "parameters":{ + "command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc", + "shell": "true" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "terminal" + ], + "evaluator": { + "func": "is_in_vm_clickboard", + "expected": { + "type": "vm_command_line", + "command": "xsel --clipboard --output", + "shell": "true" + }, + "result": { + "type": "rule", + "rules": { + "expected": "/home/user/Data3/List3/secret.docx" + } + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json new file mode 100644 index 0000000..c9c97fe --- /dev/null +++ b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json @@ -0,0 +1,50 @@ +{ + "id": "7ff48d5b-2df2-49da-b500-a5150ffc7f18", + "snapshot": "multiapps", + "instruction": "I am a Chinese citizen and I want to go to Macau to watch a concert recently, but I have not yet applied for a visa for Macau. I live in Futian District, Shenzhen City. I heard that Shenzhen currently has 24-hour self-service check-in machines. Please help me find the addresses of 5 24-hour self-service check-in machines in Futian District and save them in Chinese in this open word document.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1wrxlOrxZr3I5vaF1KRzc9nA9UcWW2ReU", + "path": "/home/user/Desktop/AllLocations.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/AllLocations.docx" + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "AllLocations.docx - LibreOffice Writer" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os", "writer" + ], + "evaluator": { + "func": "fuzzy_place_math", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/AllLocations.docx", + "dest": "AllLocations.docx" + }, + "expected": { + "type": "rule", + "rules":{ + "expected": ["正义", "海天综合大厦", "盛世家园", "香梅路", "金沙嘴大厦", "卓悦汇", "昌泰公寓", "天安数码城", "振兴路", "梅岗南街", "滨河大道", "上沙", "香蜜原著", "八卦岭", "竹园", "竹子林五路", "体育中心", "赤尾", "书城中心广场", "侨城东路", "沙头所", "水围村", "黄木岗", "华强广场", "国大药房","科学馆","江苏宾馆", "梅林"] + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json new file mode 100644 index 0000000..a5aa3a3 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json @@ -0,0 +1,66 @@ +{ + "id": "873cafdd-a581-47f6-8b33-b9696ddb7b05", + "snapshot": "multiapps", + "instruction": "My friend is a \"plugin guru\" and he recommended some good plug-ins to me. Please go to the Chrome plug-in store and install all the listed plug-ins.", + "source": "author", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1el4WdiJVpUP49FABN_DpC7yy7obgJSC5", + "path": "/home/user/Desktop/Recommended_plugin_list.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/Recommended_plugin_list.docx" + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Recommended_plugin_list.docx - LibreOffice Writer" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "chrome", "Writer" + ], + "evaluator": { + "func": "is_expected_installed_extensions", + "expected": { + "type": "rule", + "rules":{ + "expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"] + } + }, + "result": { + "type": "find_installed_extension_name" + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json new file mode 100644 index 0000000..17aac76 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json @@ -0,0 +1,88 @@ +{ + "id": "9f3bb592-209d-43bc-bb47-d77d9df56504", + "snapshot": "multiapps", + "instruction": "I downloaded a video to practice listening, but I don't know how to remove the subtitles. Please help me remove the subtitles from the video and export it as \"subtitles.srt\" and store it in the same directory as the video.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1YrPG1sXDACdXlrBlW8-RUzovWUJ0qoC5", + "path": "/home/user/video.txt" + } + ] + } + }, + { + "type": "execute", + "parameters":{ + "command":"echo lixiaochuan20 | sudo -S mv /home/user/video.txt /home/user/video.mp4", + "shell": "true" + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/video.mp4" + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "video.mp4 - VLC media player" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", "vlc" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1Q6GV39h74bc_8_kFKjNW3RMxUSgXaaWR", + "path": "/home/user/subtitles_Gold.srt" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=14eBaLVfUekkpj3dPkdDwHehHRmzx2njy", + "path": "/home/user/Desktop/subtitles_script.txt" + } + ] + } + }, + { + "type": "command", + "parameters": { + "command": "echo lixiaochuan20 | sudo -S pip install pysrt", + "shell" :"true" + } + } + ], + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "echo lixiaochuan20 | mv /home/user/Desktop/subtitles_script.txt /home/user/Desktop/subtitles_script.py && python /home/user/Desktop/subtitles_script.py", + "shell" :"true" + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "true\n" + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json b/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json new file mode 100644 index 0000000..dafe64d --- /dev/null +++ b/evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json @@ -0,0 +1,62 @@ +{ + "id": "a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a", + "snapshot": "multiapps", + "instruction": "I have developed a new Chrome extension myself, so it needs to be installed manually. Please help me install the extension located in the Desktop directory into the Chrome browser.", + "source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en", + "config": [ + { + "type": "download", + "parameters": { + "files":[ + { + "url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download", + "path":"/home/user/Desktop/helloExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S apt-get update -y && echo password | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip", + "shell": true + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os" + ], + "evaluator": { + "func": "is_in_list", + "result": { + "type": "find_unpacked_extension_path" + }, + "expected": { + "type": "rule", + "rules": { + "expected": "/home/user/Desktop/helloExtension" + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json new file mode 100644 index 0000000..0fc1380 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json @@ -0,0 +1,80 @@ +{ + "id": "ce2b64a2-ddc1-4f91-8c7d-a88be7121aac", + "snapshot": "multiapps", + "instruction": "There are several pictures of mountains in my Pictures directory, but I don’t know the names of these mountains. Please help me identify these pictures and change the names of these pictures to the names of the mountains in the pictures.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1LSFS3AI4t5aOIEUby73ZHvURqnx5cw7C", + "path": "/home/user/Pictures/picture1.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1pdRoU_FnqoDgenvp1WzFJZrsGnsexRoo", + "path": "/home/user/Pictures/picture2.jpg" + + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1ajGd9b_YvE1Ay59Pqh8RnePgbnepGc_g", + "path": "/home/user/Pictures/picture3.jpg" + + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os", "image" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1cbBn9BqtkmiZH-nNJAfQ_RYjaAUGHHTG", + "path": "/home/user/Desktop/image_script.txt" + } + ] + } + } + ], + "func": "check_direct_json_object", + "result":{ + "type": "vm_command_line", + "command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/image_script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py", + "shell": "true" + }, + "expected": { + "type": "rule", + "rules":{ + "expected":{ + "ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": "Kilimanjaro", + "999957c8de835bfa420d198270e7a6b079ee20ff53a3f214491e8134768a7c0b": "Himalayas", + "79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": "Hua" + }, + "expect_in_result": true + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json new file mode 100644 index 0000000..f7adb15 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json @@ -0,0 +1,75 @@ +{ + "id": "da922383-bfa4-4cd3-bbad-6bebab3d7742", + "snapshot": "multiapps", + "instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs you are opening now in pdf format to /home/user/Documents/Blogs.", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://lilianweng.github.io/posts/2023-06-23-agent/", + "https://lilianweng.github.io/posts/2024-02-05-human-data-quality/" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os" + ], + "evaluator": { + "postconfig":[ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=19SJ4ydB36lwIK4K4B57sl3bsY9T-3WNv", + "path": "/home/user/Desktop/script.txt" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "pip install PyMuPDF", + "shell": "true" + } + } + ], + "func": "exact_match", + "result":{ + "type": "vm_command_line", + "command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py", + "shell": "true" + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "[1, 1]\n" + } + } + } + } + \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json b/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json new file mode 100644 index 0000000..a7acd4e --- /dev/null +++ b/evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json @@ -0,0 +1,53 @@ +{ + "id": "dd60633f-2c72-42ba-8547-6f2c8cb0fdb0", + "snapshot": "multiapps", + "instruction": "I ran some simple code demos on the currently open google colab, and I think the effect is pretty good. Please help me extract the code in all code boxes, merge it into a \"task.py\" file and store it in the local Home directory.", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://colab.research.google.com/github/stanfordnlp/dspy/blob/main/intro.ipynb#scrollTo=L1BHAoL_LRd7" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", "os", "writer" + ], + "evaluator": { + "func": "compare_python_pure_text", + "result": { + "type": "vm_file", + "path": "/home/user/colab.py", + "dest": "colab.py" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?export=download&id=1Z1Vl7SOXf0yjoKttkZD1oYbLM7TgXr5s", + "dest": "colab_Gold.txt" + } + } + } + \ No newline at end of file