From dfc12430c032069ae6b90a87b310fb28a0a3bfd0 Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Mon, 29 Jan 2024 19:22:22 +0800 Subject: [PATCH] add multi-app example; drop dbt examples (examples in another project added by accident) --- desktop_env/evaluators/metrics/docs.py | 28 ++-- .../0aa56709-3293-5849-ad47-e377f49fd3a0.json | 119 ----------------- .../492c2c87-b04a-544a-b5dd-eb808036bf85.json | 89 ------------- .../49f981ee-f793-5e27-9a53-083d66934ea1.json | 126 ------------------ .../8aa9e870-b0c9-5417-be80-03154e83c7a3.json | 102 -------------- .../8ff98608-8e0e-526e-9413-d744554ba708.json | 86 ------------ .../2b9493d7-49b8-493a-a71b-56cd1f4d6908.json | 101 ++++++++++++++ .../2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json | 63 +++++++++ .../51f5801c-18b3-4f25-b0c3-02f85507a078.json | 46 +++++++ .../settings/googledrive/credentials.json | 2 +- 10 files changed, 228 insertions(+), 534 deletions(-) delete mode 100644 evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json delete mode 100644 evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json delete mode 100644 evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json delete mode 100644 evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json delete mode 100644 evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json create mode 100644 evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json create mode 100644 evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json create mode 100644 evaluation_examples/examples/multi_apps/51f5801c-18b3-4f25-b0c3-02f85507a078.json diff --git a/desktop_env/evaluators/metrics/docs.py b/desktop_env/evaluators/metrics/docs.py index 3e82e7d..8360c60 100644 --- a/desktop_env/evaluators/metrics/docs.py +++ b/desktop_env/evaluators/metrics/docs.py @@ -50,24 +50,30 @@ def contains_page_break(docx_file): return 0 -def compare_docx_files(file1, file2): +def compare_docx_files(file1, file2, ignore_blanks=True): doc1 = Document(file1) doc2 = Document(file2) doc1_paragraphs = [p.text for p in doc1.paragraphs] doc2_paragraphs = [p.text for p in doc2.paragraphs] - if len(doc1_paragraphs) != len(doc2_paragraphs): - # print(len(doc1_paragraphs)) - # print(len(doc2_paragraphs)) - return 0 - - # Compare each paragraph - for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): - if p1 != p2: - # print(p1) - # print(p2) + if ignore_blanks: + text1 = re.sub(r'\s+', ' ', '\n'.join(doc1_paragraphs)).strip() + text2 = re.sub(r'\s+', ' ', '\n'.join(doc2_paragraphs)).strip() + if text1 != text2: return 0 + else: + if len(doc1_paragraphs) != len(doc2_paragraphs): + # print(len(doc1_paragraphs)) + # print(len(doc2_paragraphs)) + return 0 + + # Compare each paragraph + for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs): + if p1 != p2: + # print(p1) + # print(p2) + return 0 return 1 diff --git a/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json b/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json deleted file mode 100644 index 4e74142..0000000 --- a/evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "id": "0aa56709-3293-5849-ad47-e377f49fd3a0", - "snapshot": "dbt", - "instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.", - "source": [ - "https://docs.getdbt.com/guides/manual-install?step=13" - ], - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969", - "path": "/home/user/projects/jaffle_shop.zip" - } - ] - } - }, - { - "type": "execute", - "parameters": { - "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] - } - }, - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", - "path": "/home/user/.dbt/profiles.yml" - } - ] - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "dbt", - "duckdb" - ], - "evaluator": { - "func": [ - "check_yaml_file", - "check_dbt_command", - "check_dbt_command", - "check_dbt_command", - "check_dbt_command" - ], - "conj": "and", - "result": [ - { - "type": "vm_file", - "path": "/home/user/projects/jaffle_shop/models/schema.yml", - "dest": "schema.yml" - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"] - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"] - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"] - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"] - } - ], - "expected": [ - { - "type": "rule", - "rules": [ - ["not_null", ["models", ["name", "customers"], "description"], ""], - ["not_null", ["models", ["name", "stg_customers"], "description"], ""], - ["not_null", ["models", ["name", "stg_orders"], "description"], ""], - ["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], - ["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"], - ["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"], - ["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."] - ] - }, - { - "type": "rule", - "rules": [ - ["contains", "dbt docs generate succeed", ""], - ["excludes", "dbt docs generate failed", ""] - ] - }, - { - "type": "rule", - "rules": [ - ["contains", "dbt docs serve succeed", ""], - ["excludes", "dbt docs serve failed", ""] - ] - }, - { - "type": "rule", - "rules": [ - ["contains", "dbt docs serve succeed", ""], - ["excludes", "dbt docs serve failed", ""] - ] - }, - { - "type": "rule", - "rules": [ - ["excludes", "error", ""], - ["excludes", "Error", ""], - ["contains", "Catalog written to", ""] - ] - } - ] - } -} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json b/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json deleted file mode 100644 index 9dbfe00..0000000 --- a/evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "id": "492c2c87-b04a-544a-b5dd-eb808036bf85", - "snapshot": "dbt", - "instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.", - "source": [ - "https://docs.getdbt.com/guides/manual-install?step=11" - ], - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467", - "path": "/home/user/projects/jaffle_shop.zip" - } - ] - } - }, - { - "type": "execute", - "parameters": { - "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] - } - }, - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", - "path": "/home/user/.dbt/profiles.yml" - } - ] - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "dbt", - "duckdb" - ], - "evaluator": { - "func": [ - "check_database", - "check_dbt_command" - ], - "conj": "and", - "result": [ - { - "type": "vm_file", - "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", - "dest": "jaffle_shop.duckdb" - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] - } - ], - "expected": [ - { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817", - "dest": "gold_jaffle_shop.duckdb" - }, - { - "type": "rule", - "rules": [ - ["contains", "of 3 OK created sql view model .*\\.stg_customers", ""], - ["contains", "of 3 OK created sql view model .*\\.stg_orders", ""], - ["contains", "3 of 3 OK created sql table model .*\\.customers", ""], - ["contains", "Completed successfully", ""], - ["contains", "PASS=3", ""], - ["contains", "TOTAL=3", ""] - ] - } - ], - "options": [ - { - "db_type": "duckdb", - "check_type": [ - "table-schema-content", - "view-schema-content" - ] - }, - {} - ] - } -} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json b/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json deleted file mode 100644 index 5b4a21e..0000000 --- a/evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "id": "49f981ee-f793-5e27-9a53-083d66934ea1", - "snapshot": "dbt", - "instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.", - "source": [ - "https://docs.getdbt.com/guides/manual-install?step=12" - ], - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507", - "path": "/home/user/projects/jaffle_shop.zip" - } - ] - } - }, - { - "type": "execute", - "parameters": { - "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] - } - }, - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", - "path": "/home/user/.dbt/profiles.yml" - } - ] - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "dbt", - "duckdb" - ], - "evaluator": { - "func": "check_dbt_command", - "result": { - "type": "dbt_test_result", - "pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"], - "path": [ - "https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168", - "https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249", - "https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799", - "https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132", - "https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860", - "https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449", - "https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524", - "https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969", - "https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430", - "https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449", - "https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100" - ], - "dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"], - "post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"] - }, - "expected": { - "type": "rule", - "rules": [ - [ - ["excludes", "Nothing to do", ""], - ["contains", "Completed successfully", ""], - ["contains", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["excludes", "Completed successfully", ""], - ["excludes", "ERROR=0", ""] - ], - [ - ["excludes", "Nothing to do", ""], - ["contains", "Completed successfully", ""], - ["contains", "ERROR=0", ""] - ] - ] - } - } -} \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json b/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json deleted file mode 100644 index 042c2c5..0000000 --- a/evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "id": "8aa9e870-b0c9-5417-be80-03154e83c7a3", - "snapshot": "dbt", - "instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.", - "source": [ - "https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#", - "https://docs.getdbt.com/guides/manual-install?step=3", - "https://docs.getdbt.com/reference/commands/seed" - ], - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569", - "path": "/home/user/projects/jaffle-shop-data.zip" - } - ] - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "dbt", - "duckdb" - ], - "evaluator": { - "func": [ - "check_yaml_file", - "check_dbt_command", - "check_database", - "check_dbt_command" - ], - "conj": "and", - "result": [ - { - "type": "dbt_profiles", - "paths": [ - "/home/user/.dbt/profiles.yml", - "/home/user/projects/jaffle_shop/profiles.yml", - "$DBT_PROFILES_DIR" - ], - "dest": "profiles.yml" - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"] - }, - { - "type": "vm_file", - "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", - "dest": "jaffle_shop.duckdb" - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"] - } - ], - "expected": [ - { - "type": "rule", - "rules": [ - ["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"], - ["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"], - ["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]], - ["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]] - ] - }, - { - "type": "rule", - "rules": [ - ["contains", "OK connection ok", ""], - ["contains", "All checks passed", ""] - ] - }, - { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476", - "dest": "gold_jaffle_shop.duckdb" - }, - { - "type": "rule", - "rules": [ - ["excludes", "Nothing to do", ""], - ["contains", "Completed successfully", ""], - ["contains", "PASS=3", ""], - ["contains", "TOTAL=3", ""] - ] - } - ], - "options": [ - {}, - {}, - { - "db_type": "duckdb", - "check_type": ["table-schema-content"] - }, - {} - ] - } - } \ No newline at end of file diff --git a/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json b/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json deleted file mode 100644 index 33117cb..0000000 --- a/evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "id": "8ff98608-8e0e-526e-9413-d744554ba708", - "snapshot": "dbt", - "instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.", - "source": [ - "https://docs.getdbt.com/guides/manual-install?step=10", - "https://docs.getdbt.com/guides/manual-install?step=8", - "https://docs.getdbt.com/guides/manual-install?step=9" - ], - "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661", - "path": "/home/user/projects/jaffle_shop.zip" - } - ] - } - }, - { - "type": "execute", - "parameters": { - "command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"] - } - }, - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799", - "path": "/home/user/.dbt/profiles.yml" - } - ] - } - } - ], - "trajectory": "trajectories/", - "related_apps": [ - "dbt", - "duckdb" - ], - "evaluator": { - "func": [ - "check_database", - "check_dbt_command" - ], - "conj": "and", - "result": [ - { - "type": "vm_file", - "path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", - "dest": "jaffle_shop.duckdb" - }, - { - "type": "vm_command_line", - "command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"] - } - ], - "expected": [ - { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749", - "dest": "gold_jaffle_shop.duckdb" - }, - { - "type": "rule", - "rules": [ - ["contains", "1 of 1 OK created sql table model .*\\.customers", ""], - ["contains", "Completed successfully", ""], - ["contains", "PASS=1", ""], - ["contains", "TOTAL=1", ""] - ] - } - ], - "options": [ - { - "db_type": "duckdb", - "check_type": ["table-schema-content"] - }, - {} - ] - } - } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json b/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json new file mode 100644 index 0000000..d83ed1b --- /dev/null +++ b/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json @@ -0,0 +1,101 @@ +{ + "id": "2b9493d7-49b8-493a-a71b-56cd1f4d6908", + "snapshot": "libreoffice_writer", + "instruction": "Hey, my LibreOffice Writer seems to have frozen and I can't get it to close normally. Can you help me force quit the application from the command line? I'm on Ubuntu and I don't want to restart my computer or lose any other work I have open.", + "source": "https://devicetests.com/kill-libreoffice-writer-command-line-ubuntu", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=104pg3yochKyH2Uvlp3BdvKmHgYmSIESu&export=download&authuser=0&confirm=t&uuid=d1926366-4e54-4a44-8dcd-fc49ed6524d7&at=APZUnTXcBFV9kcacsA0toU83lMKJ:1706505549057d", + "path": "/home/user/Desktop/15-MB-docx-file-download.docx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/15-MB-docx-file-download.docx" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "history -c && echo > ~/.bash_history && sleep 3" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gnome-terminal", + "--maximize" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_writer", + "terminal" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "killall gnome-terminal-server" + ] + } + } + ], + "func": ["check_include_exclude", "check_include_exclude"], + "conj": "and", + "result": [ + { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi" + ] + }, + { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi" + ] + } + ], + "expected": [ + { + "type": "rule", + "rules": { + "include": [ + "true\n" + ] + } + }, + { + "type": "rule", + "rules": { + "include": [ + "true\n" + ] + } + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json b/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json new file mode 100644 index 0000000..3926120 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json @@ -0,0 +1,63 @@ +{ + "id": "2c9fc0de-3ee7-45e1-a5df-c86206ad78b5", + "snapshot": "os", + "instruction": "Could you help me push the changes from commandline in current project to origin main, with the commit message \"daily update\"?", + "source": "https://nikki-ricks.medium.com/how-to-use-git-add-commit-and-push-in-vs-code-and-command-line-35c0e8c47b62", + "config": [ + { + "type": "download", + "path": "", + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git" + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gnome-terminal", + "--maximize", + "--working-directory=/home/user/projects/hello_world" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "os", + "terminal" + ], + "evaluator": { + "func": "compare_docx_files", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/notes.docx", + "dest": "notes.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download", + "dest": "notes_gold.docx" + }, + "options": { + "ignore_blanks": true + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/51f5801c-18b3-4f25-b0c3-02f85507a078.json b/evaluation_examples/examples/multi_apps/51f5801c-18b3-4f25-b0c3-02f85507a078.json new file mode 100644 index 0000000..3d32ee5 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/51f5801c-18b3-4f25-b0c3-02f85507a078.json @@ -0,0 +1,46 @@ +{ + "id": "51f5801c-18b3-4f25-b0c3-02f85507a078", + "snapshot": "libreoffice_impress", + "instruction": "I've been working on this presentation in LibreOffice Impress and I've added a bunch of speaker notes for my upcoming talk. I'd like to have those notes handy in a separate document when I rehearse. Could you assist me in extracting all the presenter notes from the Impress file and saving them as a Word document? Just keep the text of the notes, do not add any formatting or page number information. I'd like the file to be named 'notes.docx' and placed on my Desktop for easy access.", + "source": "https://github.com/danielrcollins1/ImpressExtractNotes", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1e12nL_V7bffaLSocQ86EiGCdygzggWeu&export=download", + "path": "/home/user/Desktop/Dickinson_Slides.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/Dickinson_Slides.pptx" + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_impress", + "libreoffice_writer" + ], + "evaluator": { + "func": "compare_docx_files", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/notes.docx", + "dest": "notes.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download", + "dest": "notes_gold.docx" + }, + "options": { + "ignore_blanks": true + } + } +} \ No newline at end of file diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json index 1174a00..ef8905d 100644 --- a/evaluation_examples/settings/googledrive/credentials.json +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T16:47:34Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-29T05:13:41Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file