From 5ac80dc309d9a061461cbb7e9af5ff913bb2cfe3 Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Fri, 26 Jan 2024 00:53:35 +0800 Subject: [PATCH] update examples --- desktop_env/controllers/setup.py | 2 +- desktop_env/evaluators/getters/chrome.py | 65 +++++---- .../0aa56709-3293-5849-ad47-e377f49fd3a0.json | 119 +++++++++++++++ .../492c2c87-b04a-544a-b5dd-eb808036bf85.json | 89 ++++++++++++ .../49f981ee-f793-5e27-9a53-083d66934ea1.json | 126 ++++++++++++++++ .../8aa9e870-b0c9-5417-be80-03154e83c7a3.json | 102 +++++++++++++ .../8ff98608-8e0e-526e-9413-d744554ba708.json | 86 +++++++++++ .../46407397-a7d5-4c6b-92c6-dbe038b1457b.json | 136 ++++++++++++++++++ .../b52b40a5-ad70-4c53-b5b0-5650a8387052.json | 111 ++++++++++++++ .../settings/google/settings.json | 2 +- .../settings/googledrive/credentials.json | 2 +- .../settings/thunderbird/settings.json | 5 + 12 files changed, 818 insertions(+), 27 deletions(-) create mode 100644 evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json create mode 100644 evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json create mode 100644 evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json create mode 100644 evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json create mode 100644 evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json create mode 100644 evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json create mode 100644 evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json create mode 100644 evaluation_examples/settings/thunderbird/settings.json diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index b0f9b5b..c56832b 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -536,7 +536,7 @@ class SetupController: page.goto(url) logger.info(f"Opened new page: {url}") settings = json.load(open(config['settings_file'])) - email, password = settings['account'], settings['password'] + email, password = settings['email'], settings['password'] try: page.wait_for_selector('input[type="email"]', state="visible", timeout=3000) diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 89145c1..bcfab09 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -469,34 +469,51 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str: To retrieve the file, we provide two options in config dict: 1. query: a list of queries to search the file, each query is a string that follows the format of Google Drive search query 2. path: a list of path to the file, 'folder/subfolder/filename' -> ['folder', 'subfolder', 'filename'] + 3. query_list: query extends to list to download multiple files + 4. path_list: path extends to list to download multiple files + dest: target file name or list. If *_list is used, dest should also be a list of the same length. Return the downloaded filepath locally. """ settings_file = config.get('settings_file', 'evaluation_examples/settings/googledrive/settings.json') auth = GoogleAuth(settings_file=settings_file) drive = GoogleDrive(auth) - _path = os.path.join(env.cache_dir, config['dest']) + + def get_single_file(_query, _path): + parent_id = 'root' + try: + for q in _query: + search = f'( {q} ) and "{parent_id}" in parents' + filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList() + if len(filelist) == 0: # target file not found + return None + file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one + parent_id = file['id'] + + file.GetContentFile(_path, mimetype=file['mimeType']) + except: + logger.info('[ERROR]: Failed to download the file from Google Drive') + return None + return _path if 'query' in config: - query = config['query'] - if type(query) != list: query = [query] - else: - paths = config['path'] - if type(paths) != list: paths = [paths] - query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(paths) - 1 - else f'title = {fp} and trashed = false' for idx, fp in enumerate(paths)] - parent_id = 'root' - - try: - for q in query: - search = f'( {q} ) and "{parent_id}" in parents' - filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList() - if len(filelist) == 0: # target file not found - return None - file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one - parent_id = file['id'] - - file.GetContentFile(_path, mimetype=file['mimeType']) - except: - logger.info('[ERROR]: Failed to download the file from Google Drive') - return None - return _path \ No newline at end of file + return get_single_file(config['query'], os.path.join(env.cache_dir, config['dest'])) + elif 'path' in config: + query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(config['path']) - 1 + else f'title = {fp} and trashed = false' for idx, fp in enumerate(config['path'])] + return get_single_file(query, os.path.join(env.cache_dir, config['dest'])) + elif 'query_list' in config: + _path_list = [] + assert len(config['query_list']) == len(config['dest']) + for idx, query in enumerate(config['query_list']): + dest = config['dest'][idx] + _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) + return _path_list + else: # path_list in config + _path_list = [] + assert len(config['path_list']) == len(config['dest']) + for idx, path in enumerate(config['path_list']): + query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if jdx < len(path) - 1 + else f'title = {fp} and trashed = false' for jdx, fp in enumerate(path)] + dest = config['dest'][idx] + _path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest))) + return _path_list \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json new file mode 100644 index 0000000..4e74142 --- /dev/null +++ b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json @@ -0,0 +1,119 @@ +{ + "id": "0aa56709-3293-5849-ad47-e377f49fd3a0", + "snapshot": "dbt", + "instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=13" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_yaml_file", + "check_dbt_command", + "check_dbt_command", + "check_dbt_command", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/models/schema.yml", + "dest": "schema.yml" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"] + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"] + } + ], + "expected": [ + { + "type": "rule", + "rules": [ + ["not_null", ["models", ["name", "customers"], "description"], ""], + ["not_null", ["models", ["name", "stg_customers"], "description"], ""], + ["not_null", ["models", ["name", "stg_orders"], "description"], ""], + ["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], + ["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], + ["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"], + ["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs generate succeed", ""], + ["excludes", "dbt docs generate failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs serve succeed", ""], + ["excludes", "dbt docs serve failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "dbt docs serve succeed", ""], + ["excludes", "dbt docs serve failed", ""] + ] + }, + { + "type": "rule", + "rules": [ + ["excludes", "error", ""], + ["excludes", "Error", ""], + ["contains", "Catalog written to", ""] + ] + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json new file mode 100644 index 0000000..9dbfe00 --- /dev/null +++ b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json @@ -0,0 +1,89 @@ +{ + "id": "492c2c87-b04a-544a-b5dd-eb808036bf85", + "snapshot": "dbt", + "instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=11" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["contains", "of 3 OK created sql view model .*\\.stg_customers", ""], + ["contains", "of 3 OK created sql view model .*\\.stg_orders", ""], + ["contains", "3 of 3 OK created sql table model .*\\.customers", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=3", ""], + ["contains", "TOTAL=3", ""] + ] + } + ], + "options": [ + { + "db_type": "duckdb", + "check_type": [ + "table-schema-content", + "view-schema-content" + ] + }, + {} + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json new file mode 100644 index 0000000..5b4a21e --- /dev/null +++ b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json @@ -0,0 +1,126 @@ +{ + "id": "49f981ee-f793-5e27-9a53-083d66934ea1", + "snapshot": "dbt", + "instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=12" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": "check_dbt_command", + "result": { + "type": "dbt_test_result", + "pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"], + "path": [ + "https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168", + "https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249", + "https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799", + "https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132", + "https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860", + "https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449", + "https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524", + "https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969", + "https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430", + "https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449", + "https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100" + ], + "dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"], + "post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"] + }, + "expected": { + "type": "rule", + "rules": [ + [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["excludes", "Completed successfully", ""], + ["excludes", "ERROR=0", ""] + ], + [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "ERROR=0", ""] + ] + ] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json new file mode 100644 index 0000000..042c2c5 --- /dev/null +++ b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json @@ -0,0 +1,102 @@ +{ + "id": "8aa9e870-b0c9-5417-be80-03154e83c7a3", + "snapshot": "dbt", + "instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.", + "source": [ + "https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#", + "https://docs.getdbt.com/guides/manual-install?step=3", + "https://docs.getdbt.com/reference/commands/seed" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569", + "path": "/home/user/projects/jaffle-shop-data.zip" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_yaml_file", + "check_dbt_command", + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "dbt_profiles", + "paths": [ + "/home/user/.dbt/profiles.yml", + "/home/user/projects/jaffle_shop/profiles.yml", + "$DBT_PROFILES_DIR" + ], + "dest": "profiles.yml" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"] + }, + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"] + } + ], + "expected": [ + { + "type": "rule", + "rules": [ + ["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"], + ["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"], + ["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]], + ["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]] + ] + }, + { + "type": "rule", + "rules": [ + ["contains", "OK connection ok", ""], + ["contains", "All checks passed", ""] + ] + }, + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["excludes", "Nothing to do", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=3", ""], + ["contains", "TOTAL=3", ""] + ] + } + ], + "options": [ + {}, + {}, + { + "db_type": "duckdb", + "check_type": ["table-schema-content"] + }, + {} + ] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json new file mode 100644 index 0000000..33117cb --- /dev/null +++ b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json @@ -0,0 +1,86 @@ +{ + "id": "8ff98608-8e0e-526e-9413-d744554ba708", + "snapshot": "dbt", + "instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.", + "source": [ + "https://docs.getdbt.com/guides/manual-install?step=10", + "https://docs.getdbt.com/guides/manual-install?step=8", + "https://docs.getdbt.com/guides/manual-install?step=9" + ], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661", + "path": "/home/user/projects/jaffle_shop.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", + "path": "/home/user/.dbt/profiles.yml" + } + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "dbt", + "duckdb" + ], + "evaluator": { + "func": [ + "check_database", + "check_dbt_command" + ], + "conj": "and", + "result": [ + { + "type": "vm_file", + "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", + "dest": "jaffle_shop.duckdb" + }, + { + "type": "vm_command_line", + "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] + } + ], + "expected": [ + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749", + "dest": "gold_jaffle_shop.duckdb" + }, + { + "type": "rule", + "rules": [ + ["contains", "1 of 1 OK created sql table model .*\\.customers", ""], + ["contains", "Completed successfully", ""], + ["contains", "PASS=1", ""], + ["contains", "TOTAL=1", ""] + ] + } + ], + "options": [ + { + "db_type": "duckdb", + "check_type": ["table-schema-content"] + }, + {} + ] + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json new file mode 100644 index 0000000..5475c70 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -0,0 +1,136 @@ +{ + "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b", + "snapshot": "chrome", + "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'figures' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.freerice.com/", + "https://www.hku.hk/", + "https://about.meta.com/technologies/facebook-app/" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "compare_figures", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "query_list": [ + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '1.png' and trashed = false" + ], + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '2.png' and trashed = false" + ], + [ + "title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'", + "title = '3.png' and trashed = false" + ] + ], + "dest": [ + "1.png", + "2.png", + "3.png" + ] + }, + "expected": { + "type": "cloud_file", + "path": [ + "file1", + "file2", + "file3" + ], + "dest": [ + "1_gold.png", + "2_gold.png", + "3_gold.png" + ], + "multi": true, + "gives": [0, 1, 2] + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json new file mode 100644 index 0000000..9440ddb --- /dev/null +++ b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json @@ -0,0 +1,111 @@ +{ + "id": "b52b40a5-ad70-4c53-b5b0-5650a8387052", + "snapshot": "chrome", + "instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?", + "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", + "config": [ + { + "type": "googledrive", + "parameters": { + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "operation": ["delete"], + "args": [ + { + "query": "title = 'attachment_full.pdf' and 'root' in parents", + "trash": false + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.freerice.com/", + "https://www.hku.hk/", + "https://about.meta.com/technologies/facebook-app/" + ] + } + }, + { + "type": "login", + "parameters": { + "settings_file": "evaluation_examples/settings/google/settings.json", + "platform": "googledrive" + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "compare_pdfs", + "result": { + "type": "googledrive_file", + "settings_file": "evaluation_examples/settings/googledrive/settings.yml", + "query": [ + "title = 'attachment_full.pdf' and trashed = false and 'root' in parents" + ], + "dest": "attachment_full.pdf" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805", + "dest": "attachment_full_gold.pdf" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/settings/google/settings.json b/evaluation_examples/settings/google/settings.json index a72dd31..ae70605 100644 --- a/evaluation_examples/settings/google/settings.json +++ b/evaluation_examples/settings/google/settings.json @@ -1,4 +1,4 @@ { - "account": "xlang2024anonym@gmail.com", + "email": "xlang2024anonym@gmail.com", "password": "q]wN~0iD>H:6" } \ No newline at end of file diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json index 491c002..1174a00 100644 --- a/evaluation_examples/settings/googledrive/credentials.json +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T05:51:17Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T16:47:34Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file diff --git a/evaluation_examples/settings/thunderbird/settings.json b/evaluation_examples/settings/thunderbird/settings.json new file mode 100644 index 0000000..aab6d7d --- /dev/null +++ b/evaluation_examples/settings/thunderbird/settings.json @@ -0,0 +1,5 @@ +{ + "email": "anonym-x2024@outlook.com", + "password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>", + "user": "Anonym Tester" +} \ No newline at end of file