From da0dafc32c6ea0c9c468e9ff91041acb9935a6a7 Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Wed, 6 Mar 2024 21:20:26 +0800 Subject: [PATCH] add multi-apps 5 examples by ruisheng 2024-03-06 --- desktop_env/controllers/setup.py | 2 + desktop_env/evaluators/getters/__init__.py | 5 +- desktop_env/evaluators/getters/chrome.py | 85 ++++++++++++- desktop_env/evaluators/getters/file.py | 21 +++- desktop_env/evaluators/getters/impress.py | 21 ++-- desktop_env/evaluators/metrics/__init__.py | 3 +- desktop_env/evaluators/metrics/docs.py | 14 +++ desktop_env/evaluators/metrics/general.py | 18 +++ .../22a4636f-8179-4357-8e87-d1743ece1f81.json | 108 ++++++++++++++++ .../236833a3-5704-47fc-888c-4f298f09f799.json | 67 ++++++++++ .../5990457f-2adb-467b-a4af-5c857c92d762.json | 116 ++++++++++++++++++ .../67890eb6-6ce5-4c00-9e3d-fb4972699b06.json | 110 +++++++++++++++++ .../778efd0a-153f-4842-9214-f05fc176b877.json | 89 ++++++++++++++ .../settings/googledrive/credentials.json | 1 - 14 files changed, 645 insertions(+), 15 deletions(-) create mode 100644 evaluation_examples/examples/multi_apps/22a4636f-8179-4357-8e87-d1743ece1f81.json create mode 100644 evaluation_examples/examples/multi_apps/236833a3-5704-47fc-888c-4f298f09f799.json create mode 100644 evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json create mode 100644 evaluation_examples/examples/multi_apps/67890eb6-6ce5-4c00-9e3d-fb4972699b06.json create mode 100644 evaluation_examples/examples/multi_apps/778efd0a-153f-4842-9214-f05fc176b877.json delete mode 100644 evaluation_examples/settings/googledrive/credentials.json diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 7a96288..b475008 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -450,6 +450,8 @@ class SetupController: query(str): query pattern string to search files or folder in google drive to delete, please refer to https://developers.google.com/drive/api/guides/search-files?hl=en about how to write query string. trash(bool): whether to delete files permanently or move to trash. By default, trash=false, completely delete it. + for mkdirs: + path(List[str]): the path in the google drive to create folder for upload: path(str): remote url to download file dest(List[str]): the path in the google drive to store the downloaded file diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 43f4c1b..958d98d 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -23,9 +23,10 @@ from .chrome import ( get_active_tab_url_parse, get_gotoRecreationPage_and_get_html_content, get_url_dashPart, - get_active_url_from_accessTree + get_active_url_from_accessTree, + get_info_from_website ) -from .file import get_cloud_file, get_vm_file, get_cache_file +from .file import get_cloud_file, get_vm_file, get_cache_file, get_content_from_vm_file from .general import get_vm_command_line, get_vm_terminal_output from .gimp import get_gimp_config_file from .impress import get_audio_in_slide diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 86aefc4..8bd5842 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -11,7 +11,7 @@ import lxml.etree import requests from lxml.cssselect import CSSSelector from lxml.etree import _Element -from playwright.sync_api import sync_playwright +from playwright.sync_api import sync_playwright, expect from pydrive.auth import GoogleAuth from pydrive.drive import GoogleDrive, GoogleDriveFileList, GoogleDriveFile @@ -35,6 +35,89 @@ WARNING: """ +def get_info_from_website(env, config: Dict[Any, Any]) -> Any: + """ Get information from a website. Especially useful when the information may be updated through time. + Args: + env (Any): The environment object. + config (Dict[Any, Any]): The configuration dictionary. + - url (str): The URL of the website to visit + - infos (List[Dict[str, str]]): The list of information to be extracted from the website. Each dictionary contains: + - action (str): chosen from 'inner_text', 'attribute', 'click_and_inner_text', 'click_and_attribute', etc., concretely, + - inner_text: extract the inner text of the element specified by the selector + - attribute: extract the attribute of the element specified by the selector + - click_and_inner_text: click elements following the selector and then extract the inner text of the last element + - click_and_attribute: click elements following the selector and then extract the attribute of the last element + - selector (Union[str, List[str]]): The CSS selector(s) of the element(s) to be extracted. + - attribute (str): optional for 'attribute' and 'click_and_attribute', the attribute to be extracted. + - backups (Any): The backup information to be returned if the extraction fails. + """ + try: + host = env.vm_ip + port = 9222 # fixme: this port is hard-coded, need to be changed from config file + remote_debugging_url = f"http://{host}:{port}" + with sync_playwright() as p: + # connect to remote Chrome instance + try: + browser = p.chromium.connect_over_cdp(remote_debugging_url) + except Exception as e: + # If the connection fails (e.g., the agent close the browser instance), start a new browser instance + app = 'chromium' if 'arm' in platform.machine() else 'google-chrome' + payload = json.dumps({"command": [ + app, + "--remote-debugging-port=1337" + ], "shell": False}) + headers = {"Content-Type": "application/json"} + requests.post("http://" + host + ":5000/setup" + "/launch", headers=headers, data=payload) + time.sleep(5) + browser = p.chromium.connect_over_cdp(remote_debugging_url) + + page = browser.contexts[0].new_page() + page.goto(config["url"]) + page.wait_for_load_state('load') + infos = [] + for info_dict in config.get('infos', []): + if page.url != config["url"]: + page.goto(config["url"]) + page.wait_for_load_state('load') + action = info_dict.get('action', 'inner_text') + if action == "inner_text": + ele = page.locator(info_dict['selector']) + expect(ele).to_be_visible() + infos.append(ele.inner_text()) + elif action == "attribute": + ele = page.locator(info_dict['selector']) + expect(ele).to_be_visible() + infos.append(ele.get_attribute(info_dict['attribute'])) + elif action == 'click_and_inner_text': + for idx, sel in enumerate(info_dict['selector']): + if idx != len(info_dict['selector']) - 1: + link = page.locator(sel) + expect(link).to_be_visible() + link.click() + page.wait_for_load_state('load') + else: + ele = page.locator(sel) + expect(ele).to_be_visible() + infos.append(ele.inner_text()) + elif action == 'click_and_attribute': + for idx, sel in enumerate(info_dict['selector']): + if idx != len(info_dict['selector']) - 1: + link = page.locator(sel) + expect(link).to_be_visible() + link.click() + page.wait_for_load_state('load') + else: + ele = page.locator(sel) + expect(ele).to_be_visible() + infos.append(ele.get_attribute(info_dict['attribute'])) + else: + raise NotImplementedError(f'The action {action} is not supported yet.') + return infos + except Exception as e: + logger.error(f'[ERROR]: failed to obtain information from the website: {config["url"]}. Use backup results instead.') + return config.get('backups', None) + + # The following ones just need to load info from the files of software, no need to connect to the software def get_default_search_engine(env, config: Dict[str, str]): os_type = env.vm_platform diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 95c0a18..98f6e00 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,8 +1,27 @@ import os from typing import Dict, List, Set from typing import Optional, Any, Union - import requests +import pandas as pd + + +def get_content_from_vm_file(env, config: Dict[str, Any]) -> Any: + """ + Config: + path (str): absolute path on the VM to fetch + """ + + path = config["path"] + file_path = get_vm_file(env, {"path": path, "dest": os.path.basename(path)}) + file_type, file_content = config['file_type'], config['file_content'] + if file_type == 'xlsx': + if file_content == 'last_row': + df = pd.read_excel(file_path) + last_row = df.iloc[-1] + last_row_as_list = last_row.astype(str).tolist() + return last_row_as_list + else: + raise NotImplementedError(f"File type {file_type} not supported") def get_cloud_file(env, config: Dict[str, Any]) -> Union[str, List[str]]: diff --git a/desktop_env/evaluators/getters/impress.py b/desktop_env/evaluators/getters/impress.py index ec4a154..d0673dd 100644 --- a/desktop_env/evaluators/getters/impress.py +++ b/desktop_env/evaluators/getters/impress.py @@ -40,20 +40,23 @@ def get_audio_in_slide(env, config: Dict[str, str]): audio_file_path = audio_file_path.replace('\\', '/') # Create a temporary directory to extract the audio file - with tempfile.TemporaryDirectory() as tmpdirname: - # Extract the audio file - myzip.extract(audio_file_path, tmpdirname) - # Get the full path of the extracted audio file - extracted_audio_path = os.path.join(tmpdirname, audio_file_path) - # Return the extracted audio file path - audio_file_path = extracted_audio_path - + tmpdirname = os.path.dirname(ppt_file_localhost_path) + myzip.extract(audio_file_path, tmpdirname) + audio_file_path = os.path.join(tmpdirname, audio_file_path) + return audio_file_path + # with tempfile.TemporaryDirectory() as tmpdirname: + # # Extract the audio file + # myzip.extract(audio_file_path, tmpdirname) + # # Get the full path of the extracted audio file + # extracted_audio_path = os.path.join(tmpdirname, audio_file_path) + # # Return the extracted audio file path + # audio_file_path = extracted_audio_path else: # the audio file is external to the .pptx file # Return the audio file path assert target.startswith("file://"), target audio_file_path = target[7:] - + break if audio_file_path is None: return None diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index c965a95..00165aa 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -60,7 +60,8 @@ from .general import ( fuzzy_match, check_include_exclude, check_direct_json_object, - diff_text_file + diff_text_file, + literal_match ) from .gimp import ( check_brightness_decrease_and_structure_sim, diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index b44df9c..4fa1a03 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -58,6 +58,8 @@ def contains_page_break(docx_file): def compare_docx_files(file1, file2, **options): ignore_blanks = options.get('ignore_blanks', True) + ignore_case = options.get('ignore_case', False) + ignore_order = options.get('ignore_order', False) content_only = options.get('content_only', False) def get_paragraph_texts_odt(document): @@ -82,11 +84,17 @@ def compare_docx_files(file1, file2, **options): doc2 = Document(file2) doc1_paragraphs = [p.text for p in doc1.paragraphs] doc2_paragraphs = [p.text for p in doc2.paragraphs] + if ignore_order: + doc1_paragraphs = sorted(doc1_paragraphs) + doc2_paragraphs = sorted(doc2_paragraphs) elif file1.endswith('.odt') and file2.endswith('.odt'): doc1 = load(file1) doc2 = load(file2) doc1_paragraphs = get_paragraph_texts_odt(doc1) doc2_paragraphs = get_paragraph_texts_odt(doc2) + if ignore_order: + doc1_paragraphs = sorted(doc1_paragraphs) + doc2_paragraphs = sorted(doc2_paragraphs) else: # Unsupported file types or mismatch print("Unsupported file types or mismatch between file types.") @@ -96,6 +104,8 @@ def compare_docx_files(file1, file2, **options): # Compare the content of the documents text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip() + if ignore_case: + text1, text2 = text1.lower(), text2.lower() similarity = fuzz.ratio(text1, text2) / 100.0 return similarity @@ -103,6 +113,8 @@ def compare_docx_files(file1, file2, **options): if ignore_blanks: text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip() + if ignore_case: + text1, text2 = text1.lower(), text2.lower() if text1 != text2: return 0 else: @@ -111,6 +123,8 @@ def compare_docx_files(file1, file2, **options): # Compare each paragraph for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): + if ignore_case: + p1, p2 = p1.lower(), p2.lower() if p1 != p2: return 0 diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 4458a69..b3a837d 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -39,6 +39,24 @@ def exact_match(result, rules) -> float: else: return 0. + +def literal_match(result: Any, expected: Any, **options) -> float: + literal_type = options.get('type', 'str') + if literal_type == 'str': + ignore_case = options.get('ignore_case', False) + score = str(result) == str(expected) if not ignore_case else str(result).lower() == str(expected).lower() + return float(score) + elif literal_type == 'list': + if type(result) not in [list, tuple] or type(expected) not in [list, tuple] or len(result) != len(expected): + return .0 + ignore_case = options.get('ignore_case', False) + result = [str(s) for s in result] if not ignore_case else [str(s).lower() for s in result] + expected = [str(s) for s in expected] if not ignore_case else [str(s).lower() for s in expected] + return float(result == expected) + else: + raise NotImplementedError(f"Type {type} not supported") + + def is_in_list(result, rules) -> float: expect = rules["expected"] if expect in result: diff --git a/evaluation_examples/examples/multi_apps/22a4636f-8179-4357-8e87-d1743ece1f81.json b/evaluation_examples/examples/multi_apps/22a4636f-8179-4357-8e87-d1743ece1f81.json new file mode 100644 index 0000000..d22ddaf --- /dev/null +++ b/evaluation_examples/examples/multi_apps/22a4636f-8179-4357-8e87-d1743ece1f81.json @@ -0,0 +1,108 @@ +{ + "id": "22a4636f-8179-4357-8e87-d1743ece1f81", + "snapshot": "chrome", + "instruction": "Please help me convert the file \"Meeting-Agenda.docx\" to a pdf file and upload to the folder 'meetings' in my google drive.", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": [ + "delete", + "mkdirs" + ], + "args": [ + { + "query": "title = 'Meeting-Agenda.docx' or title = 'Meeting-Agenda.pdf'", + "trash": false + }, + { + "path": [ + "meetings" + ] + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://news.google.com", + "https://x.com", + "https://www.deepl.com" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=17fkMG4guromDzAHNCzzJieZHa2nJDBpc&export=download&authuser=0&confirm=t", + "path": "/home/user/Desktop/Meeting-Agenda.docx" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "libreoffice", + "--writer", + "/home/user/Desktop/Meeting-Agenda.docx" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer", + "chrome" + ], + "evaluator": { + "func": "compare_pdfs", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "path": [ + "meetings", + "Meeting-Agenda.pdf" + ], + "dest": "Meeting-Agenda.pdf" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1wHWQ6mTJcNLtrL83GrUPzxx2HeOC4L8T&export=download&authuser=0&confirm=t", + "dest": "gold-Meeting-Agenda.pdf" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/236833a3-5704-47fc-888c-4f298f09f799.json b/evaluation_examples/examples/multi_apps/236833a3-5704-47fc-888c-4f298f09f799.json new file mode 100644 index 0000000..3c5f349 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/236833a3-5704-47fc-888c-4f298f09f799.json @@ -0,0 +1,67 @@ +{ + "id": "236833a3-5704-47fc-888c-4f298f09f799", + "snapshot": "chrome", + "instruction": "Find daily papers on Huggingface and take down all the titles, authors and the abstracts of papers on 1st March, 2024 in the doc file 'paper_reading_2024_03_01.docx' on desktop. Each paragraph (split by empty lines) conforms to the following format:\nTitle: xxx\nAuthors: xxx, xxx, xxx\nAbstract: xxxxxxxx.\nArxiv PDF: https://xxxx.pdf", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://huggingface.co/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "libreoffice", + "--writer" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", + "libreoffice_writer" + ], + "evaluator": { + "func": "compare_docx_files", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/paper_reading_2024_03_01.docx", + "dest": "paper_reading_2024_03_01.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1TUTihXD93bIlekuYy_44fmXAhI1KVol4&export=download&authuser=0&confirm=t", + "dest": "gold_paper_reading_2024_03_01.docx" + }, + "options": { + "ignore_blanks": true, + "ignore_case": true, + "ignore_order": true + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json b/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json new file mode 100644 index 0000000..9d754dd --- /dev/null +++ b/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json @@ -0,0 +1,116 @@ +{ + "id": "5990457f-2adb-467b-a4af-5c857c92d762", + "snapshot": "chrome", + "instruction": "Append one entry of AI researcher Yann LeCun from Google Scholar into an existing table researchers.xlsx.", + "source": "authors", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://arxiv.org/abs/2005.14165", + "https://wallhaven.cc/" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EbQ75SHLthiZCBqCJtO0fLXJZcKrNler&export=download&authuser=0&confirm=t", + "path": "/home/user/Desktop/researchers.xlsx" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "nautilus", + "/home/user/Desktop" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome", + "libreoffice_calc" + ], + "evaluator": { + "func": "literal_match", + "result": { + "type": "content_from_vm_file", + "path": "/home/user/Desktop/researchers.xlsx", + "file_type": "xlsx", + "file_content": "last_row" + }, + "expected": { + "type": "info_from_website", + "url": "https://scholar.google.com/citations?user=WLN3QrAAAAAJ&hl=en", + "infos": [ + { + "action": "inner_text", + "selector": "#gsc_prf_in" + }, + { + "action": "inner_text", + "selector": "#gsc_rsb_st > tbody > tr:nth-child(1) > td:nth-child(2)" + }, + { + "action": "inner_text", + "selector": "#gsc_rsb_st > tbody > tr:nth-child(2) > td:nth-child(2)" + }, + { + "action": "inner_text", + "selector": "#gsc_rsb_st > tbody > tr:nth-child(3) > td:nth-child(2)" + }, + { + "action": "inner_text", + "selector": "#gsc_a_b > tr:nth-child(1) > td.gsc_a_t > a" + }, + { + "action": "click_and_attribute", + "selector": [ + "#gsc_a_b > tr:nth-child(1) > td.gsc_a_t > a", + "#gsc_oci_title_gg > div:nth-child(1) > a" + ], + "attribute": "href" + } + ], + "backups": [ + "Yann LeCun", + "345074", + "147", + "372", + "Deep learning", + "https://creativecoding.soe.ucsc.edu/courses/cs523/slides/week3/DeepLearning_LeCun.pdf" + ] + }, + "options": { + "type": "list", + "ignore_case": true + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/67890eb6-6ce5-4c00-9e3d-fb4972699b06.json b/evaluation_examples/examples/multi_apps/67890eb6-6ce5-4c00-9e3d-fb4972699b06.json new file mode 100644 index 0000000..f414cc4 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/67890eb6-6ce5-4c00-9e3d-fb4972699b06.json @@ -0,0 +1,110 @@ +{ + "id": "67890eb6-6ce5-4c00-9e3d-fb4972699b06", + "snapshot": "libreoffice_calc", + "instruction": "I am an NLP researcher. Check out the best long paper awards of ACL from 2019 to 2022 and record the 'title', 'year', 'author list' and 'PDF link' into table best_awards_acl.xslx on the desktop. Separate authors only by commas, use offical aclanthology urls not arxiv, and sort rows by years in descending orders.", + "source": "authors", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1v5DgViUgAL771kBWy-qBddVGyjGmgFhK", + "path": "/home/user/Desktop/best_awards_acl.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/best_awards_acl.xlsx" + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://aclanthology.org/" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_calc", + "chrome" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "best_awards_acl.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/best_awards_acl.xlsx", + "dest": "best_awards_acl.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=17ORdOPl3sZGk3s4Wm0vESgImKZjdZBqX&export=download&authuser=0&confirm=t", + "dest": "gold_best_awards_acl.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/778efd0a-153f-4842-9214-f05fc176b877.json b/evaluation_examples/examples/multi_apps/778efd0a-153f-4842-9214-f05fc176b877.json new file mode 100644 index 0000000..a18657a --- /dev/null +++ b/evaluation_examples/examples/multi_apps/778efd0a-153f-4842-9214-f05fc176b877.json @@ -0,0 +1,89 @@ +{ + "id": "778efd0a-153f-4842-9214-f05fc176b877", + "snapshot": "vlc", + "instruction": "I'm using libreoffice impress to write slideshows. I found that the video being played by VLC media player had a good soundtrack. Please extract the audio to planet.wav and use it as background music for this slideshow.", + "source": "https://researchguides.case.edu/c.php?g=1286426", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1I-ArULOnZzlGkS9UyIuE8Dyuyus27iZt&export=download&authuser=0&confirm=t", + "path": "/home/user/Desktop/Minimalist_Business_Slides.pptx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1HiW-PokIfWRyRvLwlkiVKKNCB2h2bcx7&export=download&authuser=0&confirm=t", + "path": "/home/user/Desktop/planet.mp4" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/Minimalist_Business_Slides.pptx" + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "vlc", + "--repeat", + "/home/user/Desktop/planet.mp4" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress", + "vlc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Minimalist_Business_Slides.pptx - LibreOffice Impress", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "compare_audios", + "result": { + "type": "audio_in_slide", + "ppt_file_path": "/home/user/Desktop/Minimalist_Business_Slides.pptx", + "slide_index": 0, + "dest": "planet.wav" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1F_iBPLgVg-2g1LQ5rxKyCrFa9UitZ9yn&export=download&authuser=0&confirm=t", + "dest": "gold_planet.wav" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json deleted file mode 100644 index 049985c..0000000 --- a/evaluation_examples/settings/googledrive/credentials.json +++ /dev/null @@ -1 +0,0 @@ -{"access_token": "ya29.a0AfB_byCmR_-BUvipM02LYvRdhSIsjxhdK4a1tpDABrZOjTOWPkPhs4gy070hbrq0tZBeld813_HqZ4q5GTeLzddfVVTWfiHdDSLlT4Bf5_f4ZURSB53XJAdAyYeI7LcT4RwF1WoAQhtldbIBJ4rizVk7L6O3486u6e9OaCgYKARQSARISFQHGX2Mi1U-dePZ0efbg8nydUEsP9Q0171", "client_id": "786888752612-rgng5v9hcq4as7pn0b40gt9r5lekmht9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0edHIvL2N4T8ICgYIARAAGA4SNwF-L9IrKhoX-pALW0nJ18niS1Gy3Lg9wF_G1joCoGHLM8v4-WJsibIB04KjWYCp_40Cs1WS7es", "token_expiry": "2024-03-02T13:59:28Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCmR_-BUvipM02LYvRdhSIsjxhdK4a1tpDABrZOjTOWPkPhs4gy070hbrq0tZBeld813_HqZ4q5GTeLzddfVVTWfiHdDSLlT4Bf5_f4ZURSB53XJAdAyYeI7LcT4RwF1WoAQhtldbIBJ4rizVk7L6O3486u6e9OaCgYKARQSARISFQHGX2Mi1U-dePZ0efbg8nydUEsP9Q0171", "expires_in": 3599, "refresh_token": "1//0edHIvL2N4T8ICgYIARAAGA4SNwF-L9IrKhoX-pALW0nJ18niS1Gy3Lg9wF_G1joCoGHLM8v4-WJsibIB04KjWYCp_40Cs1WS7es", "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file