diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index b0f9b5b..4c637a7 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -108,7 +108,6 @@ class SetupController: cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format( uuid.uuid5(uuid.NAMESPACE_URL, url), os.path.basename(path))) - if not url or not path: raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).") @@ -536,7 +535,7 @@ class SetupController: page.goto(url) logger.info(f"Opened new page: {url}") settings = json.load(open(config['settings_file'])) - email, password = settings['account'], settings['password'] + email, password = settings['email'], settings['password'] try: page.wait_for_selector('input[type="email"]', state="visible", timeout=3000) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index a0a16e8..faa9f2e 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -4,8 +4,10 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line from .impress import get_audio_in_slide -from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper +from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config -from .vscode import get_vscode_config \ No newline at end of file +from .vscode import get_vscode_config +from .impress import get_audio_in_slide + diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 89145c1..bcfab09 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -469,34 +469,51 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str: To retrieve the file, we provide two options in config dict: 1. query: a list of queries to search the file, each query is a string that follows the format of Google Drive search query 2. path: a list of path to the file, 'folder/subfolder/filename' -> ['folder', 'subfolder', 'filename'] + 3. query_list: query extends to list to download multiple files + 4. path_list: path extends to list to download multiple files + dest: target file name or list. If *_list is used, dest should also be a list of the same length. Return the downloaded filepath locally. """ settings_file = config.get('settings_file', 'evaluation_examples/settings/googledrive/settings.json') auth = GoogleAuth(settings_file=settings_file) drive = GoogleDrive(auth) - _path = os.path.join(env.cache_dir, config['dest']) + + def get_single_file(_query, _path): + parent_id = 'root' + try: + for q in _query: + search = f'( {q} ) and "{parent_id}" in parents' + filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList() + if len(filelist) == 0: # target file not found + return None + file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one + parent_id = file['id'] + + file.GetContentFile(_path, mimetype=file['mimeType']) + except: + logger.info('[ERROR]: Failed to download the file from Google Drive') + return None + return _path if 'query' in config: - query = config['query'] - if type(query) != list: query = [query] - else: - paths = config['path'] - if type(paths) != list: paths = [paths] - query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(paths) - 1 - else f'title = {fp} and trashed = false' for idx, fp in enumerate(paths)] - parent_id = 'root' - - try: - for q in query: - search = f'( {q} ) and "{parent_id}" in parents' - filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList() - if len(filelist) == 0: # target file not found - return None - file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one - parent_id = file['id'] - - file.GetContentFile(_path, mimetype=file['mimeType']) - except: - logger.info('[ERROR]: Failed to download the file from Google Drive') - return None - return _path \ No newline at end of file + return get_single_file(config['query'], os.path.join(env.cache_dir, config['dest'])) + elif 'path' in config: + query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(config['path']) - 1 + else f'title = {fp} and trashed = false' for idx, fp in enumerate(config['path'])] + return get_single_file(query, os.path.join(env.cache_dir, config['dest'])) + elif 'query_list' in config: + _path_list = [] + assert len(config['query_list']) == len(config['dest']) + for idx, query in enumerate(config['query_list']): + dest = config['dest'][idx] + _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) + return _path_list + else: # path_list in config + _path_list = [] + assert len(config['path_list']) == len(config['dest']) + for idx, path in enumerate(config['path_list']): + query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if jdx < len(path) - 1 + else f'title = {fp} and trashed = false' for jdx, fp in enumerate(path)] + dest = config['dest'][idx] + _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) + return _path_list \ No newline at end of file diff --git a/desktop_env/evaluators/getters/info.py b/desktop_env/evaluators/getters/info.py index 3d41ee5..0c88fd2 100644 --- a/desktop_env/evaluators/getters/info.py +++ b/desktop_env/evaluators/getters/info.py @@ -18,3 +18,7 @@ def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]: f.write(content) return _path + + +def get_list_directory(env, config: dict) -> dict: + return env.controller.get_vm_directory_tree(config["path"]) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index b2214fd..f368699 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -4,7 +4,9 @@ from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_co from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \ evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \ - check_no_duplicates + check_no_duplicates, compare_init_lines +from .general import exact_match, fuzzy_match + from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \ compare_insert_equation, compare_highlighted_text from .docs import is_first_line_centered, check_file_exists, compare_contains_image @@ -19,10 +21,15 @@ from .slides import check_presenter_console_disable, check_image_stretch_and_cen # from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations from .table import compare_table from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter + +from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ + compare_videos +from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \ check_qt_max_volume, \ check_qt_slider_colours, check_global_key_play_pause from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \ check_json_keybindings -from .os import check_gnome_favorite_apps +from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs + diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index a3532d0..a45c300 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -56,15 +56,34 @@ def compare_docx_files(file1, file2): doc2_paragraphs = [p.text for p in doc2.paragraphs] if len(doc1_paragraphs) != len(doc2_paragraphs): + # print(len(doc1_paragraphs)) + # print(len(doc2_paragraphs)) return 0 # Compare each paragraph for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): if p1 != p2: + # print(p1) + # print(p2) return 0 return 1 +def compare_init_lines(file1, file2): + doc1 = Document(file1) + doc2 = Document(file2) + + doc1_paragraphs = [p.text for p in doc1.paragraphs] + doc2_paragraphs = [p.text for p in doc2.paragraphs] + + # Compare each paragraph + for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): + if p1 != p2: + # print(p1) + # print(p2) + return 0 + + return 1 def compare_docx_tables(docx_file1, docx_file2): doc1 = Document(docx_file1) @@ -93,6 +112,8 @@ def compare_docx_tables(docx_file1, docx_file2): def compare_line_spacing(docx_file1, docx_file2): + if not compare_docx_files(docx_file1, docx_file2): + return 0 doc1 = Document(docx_file1) doc2 = Document(docx_file2) @@ -200,8 +221,10 @@ def compare_contains_image(docx_file1, docx_file2): # print(find_default_font("Ani", config_path)) -def evaluate_colored_words_in_tables(file_path): - document = Document(file_path) +def evaluate_colored_words_in_tables(file_path1, file_path2): + if not compare_docx_files(file_path1, file_path2): + return 0 + document = Document(file_path1) for table in document.tables: # Iterate through rows and cells in the table @@ -221,8 +244,10 @@ def evaluate_colored_words_in_tables(file_path): return 1 # All words in tables are correctly colored -def check_highlighted_words(file_path): - document = Document(file_path) +def check_highlighted_words(file_path1, file_path2): + if not compare_docx_files(file_path1, file_path2): + return 0 + document = Document(file_path1) for paragraph in document.paragraphs: for run in paragraph.runs: @@ -232,8 +257,10 @@ def check_highlighted_words(file_path): return 1 # No highlighted words found -def evaluate_strike_through_last_paragraph(file_path): - document = Document(file_path) +def evaluate_strike_through_last_paragraph(file_path1, file_path2): + if not compare_docx_files(file_path1, file_path2): + return 0 + document = Document(file_path1) # Get the last paragraph last_paragraph = document.paragraphs[-1] @@ -278,8 +305,10 @@ def evaluate_spacing(file_path): return 0 -def check_italic_font_size_14(path): - document = Document(path) +def check_italic_font_size_14(path1, path2): + if not compare_docx_files(path1, path2): + return 0 + document = Document(path1) for paragraph in document.paragraphs: for run in paragraph.runs: if run.italic: @@ -365,6 +394,8 @@ def compare_docx_lines(file1, file2): doc2 = Document(file2) doc2_lines = [p.text.strip() for p in doc2.paragraphs if p.text.strip()] + # print(doc1_lines) + # print(doc2_lines) # Convert the list of lines to sets and compare return set(doc1_lines) == set(doc2_lines) diff --git a/desktop_env/evaluators/metrics/os.py b/desktop_env/evaluators/metrics/os.py index 3386466..55688fb 100644 --- a/desktop_env/evaluators/metrics/os.py +++ b/desktop_env/evaluators/metrics/os.py @@ -17,54 +17,43 @@ def check_gnome_favorite_apps(apps_str: str, rule): return 0 -# TODO: log in to the system before running this -def is_logout_successful(): - try: - subprocess.run(["whoami"]) - return 0 # Task not successful - except subprocess.CalledProcessError: - return 1 # Task successful +def is_utc_0(timedatectl_output): + """ + Format as: + Local time: Thu 2024-01-25 12:56:06 WET + Universal time: Thu 2024-01-25 12:56:06 UTC + RTC time: Thu 2024-01-25 12:56:05 + Time zone: Atlantic/Faroe (WET, +0000) +System clock synchronized: yes + NTP service: inactive + RTC in local TZ: no + """ + + utc_line = timedatectl_output.split("\n")[3] + + if utc_line.endswith("+0000)"): + return 1 + else: + return 0 -def is_battery_percentage_displayed(): - # GNOME schema and key for the setting - schema = "org.gnome.desktop.interface" - key = "show-battery-percentage" - - try: - # use gsettings to get the current setting - result = subprocess.run(['gsettings', 'get', schema, key], capture_output=True, text=True) - # examine the output is 'true' - if result.stdout.strip() == 'true': - return 1. - else: - return 0. - except Exception as e: - print(f"An error occurred: {e}") - return 0. +def check_text_enlarged(scaling_factor_str): + scaling_factor = float(scaling_factor_str) + if scaling_factor > 1.0: + return 1 + else: + return 0 -def check_auto_lock_settings(): - # The schema and keys for the GNOME desktop lock settings - schema = "org.gnome.desktop.screensaver" - lock_enabled_key = "lock-enabled" - lock_delay_key = "lock-delay" +def check_moved_jpgs(directory_list, rule): - try: - # Check if the screen lock is enabled - lock_enabled = subprocess.run(['gsettings', 'get', schema, lock_enabled_key], capture_output=True, text=True) - is_lock_enabled = lock_enabled.stdout.strip() == 'true' + expected_jpgs = rule["expected"] + moved_jpgs = [node['name'] for node in directory_list['children']] - # Check the delay before the screen is locked - lock_delay = subprocess.run(['gsettings', 'get', schema, lock_delay_key], capture_output=True, text=True) - # Extract the numerical value from the output - delay_seconds = lock_delay + if len(moved_jpgs) != len(expected_jpgs): + return 0 - # Print the results - if is_lock_enabled: - return 1. - else: - return 0. - - except Exception as e: - return 0. + if set(moved_jpgs) == set(expected_jpgs): + return 1 + else: + return 0 diff --git a/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json new file mode 100644 index 0000000..4e74142 --- /dev/null +++ b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json @@ -0,0 +1,119 @@ +{ + "id": "0aa56709-3293-5849-ad47-e377f49fd3a0", + "snapshot": "dbt", + "instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=13" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_yaml_file", + "check_dbt_command", + "check_dbt_command", + "check_dbt_command", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/models/schema.yml", + "dest": "schema.yml" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"] + } + ], + "expected": [ + { + "type": "rule", + "rules": [ + ["not_null", ["models", ["name", "customers"], "description"], ""], + ["not_null", ["models", ["name", "stg_customers"], "description"], ""], + ["not_null", ["models", ["name", "stg_orders"], "description"], ""], + ["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], + ["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], + ["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"], + ["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs generate succeed", ""], + ["excludes", "dbt docs generate failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs serve succeed", ""], + ["excludes", "dbt docs serve failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs serve succeed", ""], + ["excludes", "dbt docs serve failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["excludes", "error", ""], + ["excludes", "Error", ""], + ["contains", "Catalog written to", ""] + ] + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json new file mode 100644 index 0000000..9dbfe00 --- /dev/null +++ b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json @@ -0,0 +1,89 @@ +{ + "id": "492c2c87-b04a-544a-b5dd-eb808036bf85", + "snapshot": "dbt", + "instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=11" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["contains", "of 3 OK created sql view model .*\\.stg_customers", ""], + ["contains", "of 3 OK created sql view model .*\\.stg_orders", ""], + ["contains", "3 of 3 OK created sql table model .*\\.customers", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=3", ""], + ["contains", "TOTAL=3", ""] + ] + } + ], + "options": [ + { + "db_type": "duckdb", + "check_type": [ + "table-schema-content", + "view-schema-content" + ] + }, + {} + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json new file mode 100644 index 0000000..5b4a21e --- /dev/null +++ b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json @@ -0,0 +1,126 @@ +{ + "id": "49f981ee-f793-5e27-9a53-083d66934ea1", + "snapshot": "dbt", + "instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=12" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": "check_dbt_command", + "result": { + "type": "dbt_test_result", + "pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"], + "path": [ + "https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168", + "https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249", + "https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799", + "https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132", + "https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860", + "https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449", + "https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524", + "https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969", + "https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430", + "https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449", + "https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100" + ], + "dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"], + "post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"] + }, + "expected": { + "type": "rule", + "rules": [ + [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "ERROR=0", ""] + ] + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json new file mode 100644 index 0000000..042c2c5 --- /dev/null +++ b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json @@ -0,0 +1,102 @@ +{ + "id": "8aa9e870-b0c9-5417-be80-03154e83c7a3", + "snapshot": "dbt", + "instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.", + "source": [ + "https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#", + "https://docs.getdbt.com/guides/manual-install?step=3", + "https://docs.getdbt.com/reference/commands/seed" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569", + "path": "/home/user/projects/jaffle-shop-data.zip" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_yaml_file", + "check_dbt_command", + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "dbt_profiles", + "paths": [ + "/home/user/.dbt/profiles.yml", + "/home/user/projects/jaffle_shop/profiles.yml", + "$DBT_PROFILES_DIR" + ], + "dest": "profiles.yml" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"] + }, + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"] + } + ], + "expected": [ + { + "type": "rule", + "rules": [ + ["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"], + ["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"], + ["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]], + ["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "OK connection ok", ""], + ["contains", "All checks passed", ""] + ] + }, + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=3", ""], + ["contains", "TOTAL=3", ""] + ] + } + ], + "options": [ + {}, + {}, + { + "db_type": "duckdb", + "check_type": ["table-schema-content"] + }, + {} + ] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json new file mode 100644 index 0000000..33117cb --- /dev/null +++ b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json @@ -0,0 +1,86 @@ +{ + "id": "8ff98608-8e0e-526e-9413-d744554ba708", + "snapshot": "dbt", + "instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=10", + "https://docs.getdbt.com/guides/manual-install?step=8", + "https://docs.getdbt.com/guides/manual-install?step=9" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["contains", "1 of 1 OK created sql table model .*\\.customers", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=1", ""], + ["contains", "TOTAL=1", ""] + ] + } + ], + "options": [ + { + "db_type": "duckdb", + "check_type": ["table-schema-content"] + }, + {} + ] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index a37953d..fbf8b33 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -19,8 +19,9 @@ "type": "open", "parameters": { "path": "/home/user/OrderId_Month_Chart.xlsx" - ] - }, + } + } + ], "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ "libreoffice calc" @@ -77,4 +78,4 @@ ] } } -} +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json new file mode 100644 index 0000000..ed98840 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json @@ -0,0 +1,79 @@ +{ + "id": "39aa4e37-dc91-482e-99af-132a612d40f3", + "snapshot": "libreoffice_calc", + "instruction": "Help me rename sheet1 \"LARS_Science_Assessment\"", + "source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1Y9KUHsACajMU1kWz_XjW0VBO3RleDBXw&export=download&authuser=0&confirm=t&uuid=73fe859e-7fe1-449b-83cd-2ddaab067f90&at=APZUnTW-Gp9kn8VhJVCXi5VWxjMT:1706099595303", + "path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx" + } + } + ], + "trajectory": "trajectories/39aa4e37-dc91-482e-99af-132a612d40f3", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx", + "dest": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1JC8iKt5_XjhksBDSePi0THwrfM5oF-5A&export=download&authuser=0&confirm=t&uuid=6124298b-1487-4a85-b7d9-d9ba133c26e0&at=APZUnTVfW7o4XruPUft-Dph7um6J:1706099593830", + "dest": "LARS_Science_Assessment_Resource_List_7-30-21_gold.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_name" + }, + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json similarity index 100% rename from evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup rename to evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json diff --git a/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json b/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json index 4a20b1e..3ce8e7f 100644 --- a/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json +++ b/evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json @@ -27,7 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": "compare_docx_files", + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "04 CHIN9505 EBook Purchasing info 2021 Jan.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], + "func": "compare_init_lines", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1yyHGj8KUHDMsZmc1QeJ1KkvSEGy83jMR&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json b/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json index a25712b..3280355 100644 --- a/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json +++ b/evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=df2bb3c3-e75e-4cbc-a6bc-24120512a4e1&at=APZUnTUak54ZfgQDNxbt_PqBvNQu:1706017722797", + "url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=845f9616-2fb7-476a-abab-8b620d482ac2&at=APZUnTXB71PxHF7Dq9TC2OL_cRLm:1706199789147", "path": "Desktop/sample-recruitment-phone-script.docx" } ] @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "compare_docx_files", - "check_highlighted_words" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "sample-recruitment-phone-script.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "check_highlighted_words", "expected": { "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=1s9Dsy66-zxbCAgeTyCh0P7AT7P4jF6o3&export=download&authuser=0&confirm=t&uuid=1239f2a1-8c86-45a4-8e7d-36388ac22a69&at=APZUnTVZQzXQAMNsKKQzOw5ppT8A:1706017721589", diff --git a/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json b/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json index 9bb14b7..aaa6dae 100644 --- a/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json +++ b/evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "HK_train_record.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_lines", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json b/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json index c21d2ad..b0b0532 100644 --- a/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json +++ b/evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "GEOG2169_Course_Outline_2022-23.docx" + "path": "Desktop/GEOG2169_Course_Outline_2022-23.docx" } } ], @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "evaluate_strike_through_last_paragraph", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "GEOG2169_Course_Outline_2022-23.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "evaluate_strike_through_last_paragraph", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1IpAnQRYo1whrnzIGyo8UldZf4Tli-yVT&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json b/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json index 055f65e..b941cbc 100644 --- a/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json +++ b/evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "evaluate_colored_words_in_tables", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Dolch_Sight_Words_Primer.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "evaluate_colored_words_in_tables", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1ksn444K17lFOdm5pELrQYvuZHkOsKq69&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json b/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json index 9a75fcf..863fd45 100644 --- a/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json +++ b/evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "CCCH9003_Tutorial_guidelines.docx" + "path": "Desktop/CCCH9003_Tutorial_guidelines.docx" } } ], @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "CCCH9003_Tutorial_guidelines.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_files", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json b/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json index 6464762..50a3cfd 100644 --- a/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json +++ b/evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "CCHU9045_Course_Outline_2019-20.docx" + "path": "Desktop/CCHU9045_Course_Outline_2019-20.docx" } } ], @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "compare_line_spacing", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "CCHU9045_Course_Outline_2019-20.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "compare_line_spacing", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=16LN7uYSSXk_xwgc4IZXnN2Z1nCmPJfLm&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json b/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json index 3200a06..de4479b 100644 --- a/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json +++ b/evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "presentation_instruction_2023_Feb.docx" + "path": "Desktop/presentation_instruction_2023_Feb.docx" } } ], @@ -28,6 +28,31 @@ ], "evaluator": { "func": "compare_docx_files", + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "presentation_instruction_2023_Feb.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1bB1N2TWN0puZ6DwUFS_TDjvRWchaGP9T&export=download", diff --git a/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json b/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json index 462bee9..bce074c 100644 --- a/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json +++ b/evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json @@ -27,10 +27,32 @@ "libreoffice_writer" ], "evaluator": { - "func": [ - "check_italic_font_size_14", - "compare_docx_files" + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Y22-2119-assign4.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } ], + "func": "check_italic_font_size_14", "expected": { "type": "cloud_file", "path": "https://drive.google.com/uc?id=1GTZ-DkMxpdYx38z_s0ab85Ejgxv3qfEp&export=download", diff --git a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json new file mode 100644 index 0000000..5475c70 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -0,0 +1,136 @@ +{ + "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b", + "snapshot": "chrome", + "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'figures' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.freerice.com/", + "https://www.hku.hk/", + "https://about.meta.com/technologies/facebook-app/" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "compare_figures", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "query_list": [ + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '1.png' and trashed = false" + ], + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '2.png' and trashed = false" + ], + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '3.png' and trashed = false" + ] + ], + "dest": [ + "1.png", + "2.png", + "3.png" + ] + }, + "expected": { + "type": "cloud_file", + "path": [ + "file1", + "file2", + "file3" + ], + "dest": [ + "1_gold.png", + "2_gold.png", + "3_gold.png" + ], + "multi": true, + "gives": [0, 1, 2] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json new file mode 100644 index 0000000..9440ddb --- /dev/null +++ b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json @@ -0,0 +1,111 @@ +{ + "id": "b52b40a5-ad70-4c53-b5b0-5650a8387052", + "snapshot": "chrome", + "instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'attachment_full.pdf' and 'root' in parents", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.freerice.com/", + "https://www.hku.hk/", + "https://about.meta.com/technologies/facebook-app/" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "compare_pdfs", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "query": [ + "title = 'attachment_full.pdf' and trashed = false and 'root' in parents" + ], + "dest": "attachment_full.pdf" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805", + "dest": "attachment_full_gold.pdf" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json index ee41922..654a749 100644 --- a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json +++ b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json @@ -1,17 +1,101 @@ { - "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", - "snapshot": "os", - "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", - "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "snapshot": "os", + "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", + "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos && mkdir -p ~/Desktop/cpjpg", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos/vacation && mkdir -p ~/Desktop/photos/family && mkdir -p ~/Desktop/photos/events", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos/vacation/thailand && mkdir -p ~/Desktop/photos/vacation/hk", + "shell": true + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1H7R6flpv6vgyBOHurzR0CyvPRIobuIpI&export=download&authuser=0&confirm=t&uuid=1b41e87d-7a84-4aa6-af40-cf940e80d956&at=APZUnTWus2n1pP6l2wmIyXkCUHGs:1706196492307", + "path": "/home/user/Desktop/photos/vacation/thailand/monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1Mh3kIViHaayxNC88TpgpjbmIEl17LA3d&export=download&authuser=0&confirm=t&uuid=a8355b84-aa2e-4502-8d0f-3723ffe7d442&at=APZUnTW4nzDFgDgNeERMRKEQ4bTU:1706196493503", + "path": "/home/user/Desktop/photos/vacation/hk/hong-kong-china.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EhmbZ3QP_mdvvppC5zKA3o--Ce2F-ocO&export=download&authuser=0&confirm=t&uuid=70e05133-dcac-4168-b555-82da15a10d58&at=APZUnTXJ371jG7qiPLHBLt2zy7ij:1706196494612", + "path": "/home/user/Desktop/photos/vacation/hk/hk_group_photo.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1p_r_BMCLFG872uH9Qa-sLnqXppHvDx95&export=download&authuser=0&confirm=t&uuid=af59d107-8e46-4d2c-b72a-e4b42655959a&at=APZUnTU2hBidSnefJuUQTyJCEVFx:1706196491030", + "path": "/home/user/Desktop/photos/family/us_3.png" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1YxryNHfLPcFODnUh31CYGHGj3ZFRhvbA&export=download&authuser=0&confirm=t&uuid=0925ae33-ab33-42b9-901c-d90f11e0afcd&at=APZUnTVey7VeC9zYOmTwl0x02Iwf:1706196495770", + "path": "/home/user/Desktop/photos/events/emnlp2023.jpg" + } + ] } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "check_moved_jpgs", + "result": { + "type": "list_directory", + "path": "/home/user/Desktop/cpjpg" + }, + "expected": { + "type": "rule", + "rules": { + "expected": ["emnlp2023.jpg", "hk_group_photo.jpg", "hong-kong-china.jpg", "monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"] + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json index 9c67040..99df0f7 100644 --- a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json +++ b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json @@ -1,7 +1,7 @@ { "id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8", "snapshot": "os", - "instruction": "I want to make the text on the screen larger", + "instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?", "source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en", "trajectory": "trajectories/", "config": [], @@ -9,10 +9,11 @@ "os" ], "evaluator": { - "func": "", + "func": "check_text_enlarged", "result": { - }, - "expected": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface text-scaling-factor", + "shell": true } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json index 5482ee2..ebbb114 100644 --- a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json +++ b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json @@ -1,7 +1,7 @@ { "id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63", "snapshot": "os", - "instruction": "I want to change my system language to Chinese(simplified). Can you help me?", + "instruction": "I want to change my system language to Chinese (simplified). Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN", "trajectory": "trajectories/", "config": [], diff --git a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json index b6c00af..7ce6f21 100644 --- a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json +++ b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json @@ -1,18 +1,44 @@ { "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", "snapshot": "os", - "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?", + "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a poster of party night. Could you help me recover it from the Trash?", "source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1XaTnC_lLbR_tGTz8tcN2Tp6cNrMlNW3R&export=download&authuser=0&confirm=t&uuid=89e69a23-43cf-4316-833a-fb9d3e281460&at=APZUnTWn5zZTH4GlClO6lV1i4WwP:1706184669922", + "path": "poster_party_night.webp" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/", + "shell": true + } + } + ], "related_apps": [ "os" ], "evaluator": { - "func": "", - "result": { - }, - "expected": { - } + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "[ -f /home/user/Desktop/poster_party_night.webp ] && echo 'File exists.' || echo 'File does not exist.'", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "File exists.\n" + } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json index 1d253f4..673163b 100644 --- a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json +++ b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json @@ -4,15 +4,60 @@ "instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?", "source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S apt-get install zsh -y", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S chsh -s $(which zsh)", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S loginctl terminate-user user", + "shell": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 10 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(2); pyautogui.press(\"enter\"); pyautogui.type(\"password\"); time.sleep(2); pyautogui.press(\"enter\");" + ] + } + } + ], "related_apps": [ "os" ], "evaluator": { - "func": "", - "result": { - }, - "expected": { - } + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "echo $SHELL", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "/bin/bash\n" + } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json index 9b6cd0f..3675d05 100644 --- a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json +++ b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json @@ -1,17 +1,24 @@ { - "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", - "snapshot": "os", - "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", + "snapshot": "os", + "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.screensaver lock-enabled", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "expected": "true\n" } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json index 7b43485..46964d3 100644 --- a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json +++ b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json @@ -4,15 +4,17 @@ "instruction": "I want to set my current time zone to UTC+0. Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + ], "related_apps": [ "os" ], "evaluator": { - "func": "", + "func": "is_utc_0", "result": { - }, - "expected": { + "type": "vm_command_line", + "command": "timedatectl status", + "shell": true } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json index b50c3a9..c54f368 100644 --- a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json +++ b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json @@ -1,17 +1,36 @@ { - "id": "ddc75b62-7311-4af8-bfb3-859558542b36", - "snapshot": "os", - "instruction": "I want to uninstall the Firefox on my system. Can you help me", - "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" + "id": "ddc75b62-7311-4af8-bfb3-859558542b36", + "snapshot": "os", + "instruction": "I want to uninstall the Mahjongg on my system. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "sleep", + "parameters": { + "seconds": 1 + } + } ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "func": "check_include_exclude", + "result": { + "type": "vm_command_line", + "command": "dpkg -l | grep mahjongg", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + ], + "exclude": [ + "mahjongg" + ] } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json index 0563b9a..c9e41d8 100644 --- a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json +++ b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json @@ -1,17 +1,33 @@ { - "id": "e0df059f-28a6-4169-924f-b9623e7184cc", - "snapshot": "os", - "instruction": "I have a directory named \"test1\". Can you help me change its name into \"test2\"?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "e0df059f-28a6-4169-924f-b9623e7184cc", + "snapshot": "os", + "instruction": "I have a directory named \"todo_list_Jan_1\". Can you help me change its name into \"todo_list_Jan_2\"?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1", + "shell": true } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "[ -d ~/Desktop/todo_list_Jan_2 ] && echo 'Directory exists.' || echo 'Directory does not exist.'", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "Directory exists.\n" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json index b5ee6cd..c613775 100644 --- a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json +++ b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json @@ -8,10 +8,17 @@ "os" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface show-battery-percentage", + "shell": true }, "expected": { + "type": "rule", + "rules":{ + "expected": "true\n" + } } } } \ No newline at end of file diff --git a/evaluation_examples/settings/google/settings.json b/evaluation_examples/settings/google/settings.json index a72dd31..ae70605 100644 --- a/evaluation_examples/settings/google/settings.json +++ b/evaluation_examples/settings/google/settings.json @@ -1,4 +1,4 @@ { - "account": "xlang2024anonym@gmail.com", + "email": "xlang2024anonym@gmail.com", "password": "q]wN~0iD>H:6" } \ No newline at end of file diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json index 491c002..1174a00 100644 --- a/evaluation_examples/settings/googledrive/credentials.json +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T05:51:17Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T16:47:34Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file diff --git a/evaluation_examples/settings/thunderbird/settings.json b/evaluation_examples/settings/thunderbird/settings.json new file mode 100644 index 0000000..aab6d7d --- /dev/null +++ b/evaluation_examples/settings/thunderbird/settings.json @@ -0,0 +1,5 @@ +{ + "email": "anonym-x2024@outlook.com", + "password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>", + "user": "Anonym Tester" +} \ No newline at end of file