lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update examples

Browse Source
This commit is contained in:
rhythmcao
2024-01-26 00:53:35 +08:00
parent 8de14f4cbc
commit 5ac80dc309
12 changed files with 818 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/controllers/setup.py
View File

@@ -536,7 +536,7 @@ class SetupController:
page.goto(url)
logger.info(f"Opened new page: {url}")
settings = json.load(open(config['settings_file']))
email, password = settings['account'], settings['password']
email, password = settings['email'], settings['password']
try:
page.wait_for_selector('input[type="email"]', state="visible", timeout=3000)

65
desktop_env/evaluators/getters/chrome.py
View File

@@ -469,34 +469,51 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str:
To retrieve the file, we provide two options in config dict:
1. query: a list of queries to search the file, each query is a string that follows the format of Google Drive search query
2. path: a list of path to the file, 'folder/subfolder/filename' -> ['folder', 'subfolder', 'filename']
3. query_list: query extends to list to download multiple files
4. path_list: path extends to list to download multiple files
dest: target file name or list. If *_list is used, dest should also be a list of the same length.
Return the downloaded filepath locally.
"""
settings_file = config.get('settings_file', 'evaluation_examples/settings/googledrive/settings.json')
auth = GoogleAuth(settings_file=settings_file)
drive = GoogleDrive(auth)
_path = os.path.join(env.cache_dir, config['dest'])
def get_single_file(_query, _path):
parent_id = 'root'
try:
for q in _query:
search = f'( {q} ) and "{parent_id}" in parents'
filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList()
if len(filelist) == 0: # target file not found
return None
file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one
parent_id = file['id']
file.GetContentFile(_path, mimetype=file['mimeType'])
except:
logger.info('[ERROR]: Failed to download the file from Google Drive')
return None
return _path
if 'query' in config:
query = config['query']
if type(query) != list: query = [query]
else:
paths = config['path']
if type(paths) != list: paths = [paths]
query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(paths) - 1
else f'title = {fp} and trashed = false' for idx, fp in enumerate(paths)]
parent_id = 'root'
try:
for q in query:
search = f'( {q} ) and "{parent_id}" in parents'
filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList()
if len(filelist) == 0: # target file not found
return None
file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one
parent_id = file['id']
file.GetContentFile(_path, mimetype=file['mimeType'])
except:
logger.info('[ERROR]: Failed to download the file from Google Drive')
return None
return _path
return get_single_file(config['query'], os.path.join(env.cache_dir, config['dest']))
elif 'path' in config:
query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(config['path']) - 1
else f'title = {fp} and trashed = false' for idx, fp in enumerate(config['path'])]
return get_single_file(query, os.path.join(env.cache_dir, config['dest']))
elif 'query_list' in config:
_path_list = []
assert len(config['query_list']) == len(config['dest'])
for idx, query in enumerate(config['query_list']):
dest = config['dest'][idx]
_path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest)))
return _path_list
else: # path_list in config
_path_list = []
assert len(config['path_list']) == len(config['dest'])
for idx, path in enumerate(config['path_list']):
query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if jdx < len(path) - 1
else f'title = {fp} and trashed = false' for jdx, fp in enumerate(path)]
dest = config['dest'][idx]
_path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest)))
return _path_list

119
evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json Normal file
View File

@@ -0,0 +1,119 @@
{
"id": "0aa56709-3293-5849-ad47-e377f49fd3a0",
"snapshot": "dbt",
"instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=13"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_yaml_file",
"check_dbt_command",
"check_dbt_command",
"check_dbt_command",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/models/schema.yml",
"dest": "schema.yml"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"]
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"]
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"]
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"]
}
],
"expected": [
{
"type": "rule",
"rules": [
["not_null", ["models", ["name", "customers"], "description"], ""],
["not_null", ["models", ["name", "stg_customers"], "description"], ""],
["not_null", ["models", ["name", "stg_orders"], "description"], ""],
["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"],
["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"],
["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"],
["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."]
]
},
{
"type": "rule",
"rules": [
["contains", "dbt docs generate succeed", ""],
["excludes", "dbt docs generate failed", ""]
]
},
{
"type": "rule",
"rules": [
["contains", "dbt docs serve succeed", ""],
["excludes", "dbt docs serve failed", ""]
]
},
{
"type": "rule",
"rules": [
["contains", "dbt docs serve succeed", ""],
["excludes", "dbt docs serve failed", ""]
]
},
{
"type": "rule",
"rules": [
["excludes", "error", ""],
["excludes", "Error", ""],
["contains", "Catalog written to", ""]
]
}
]
}
}

89
evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json Normal file
View File

@@ -0,0 +1,89 @@
{
"id": "492c2c87-b04a-544a-b5dd-eb808036bf85",
"snapshot": "dbt",
"instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=11"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_database",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"dest": "jaffle_shop.duckdb"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"]
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817",
"dest": "gold_jaffle_shop.duckdb"
},
{
"type": "rule",
"rules": [
["contains", "of 3 OK created sql view model .*\\.stg_customers", ""],
["contains", "of 3 OK created sql view model .*\\.stg_orders", ""],
["contains", "3 of 3 OK created sql table model .*\\.customers", ""],
["contains", "Completed successfully", ""],
["contains", "PASS=3", ""],
["contains", "TOTAL=3", ""]
]
}
],
"options": [
{
"db_type": "duckdb",
"check_type": [
"table-schema-content",
"view-schema-content"
]
},
{}
]
}
}

126
evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json Normal file
View File

@@ -0,0 +1,126 @@
{
"id": "49f981ee-f793-5e27-9a53-083d66934ea1",
"snapshot": "dbt",
"instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=12"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": "check_dbt_command",
"result": {
"type": "dbt_test_result",
"pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"],
"path": [
"https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168",
"https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249",
"https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799",
"https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132",
"https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860",
"https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449",
"https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524",
"https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969",
"https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430",
"https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449",
"https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100"
],
"dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"],
"post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"]
},
"expected": {
"type": "rule",
"rules": [
[
["excludes", "Nothing to do", ""],
["contains", "Completed successfully", ""],
["contains", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["contains", "Completed successfully", ""],
["contains", "ERROR=0", ""]
]
]
}
}
}

102
evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json Normal file
View File

@@ -0,0 +1,102 @@
{
"id": "8aa9e870-b0c9-5417-be80-03154e83c7a3",
"snapshot": "dbt",
"instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.",
"source": [
"https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#",
"https://docs.getdbt.com/guides/manual-install?step=3",
"https://docs.getdbt.com/reference/commands/seed"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569",
"path": "/home/user/projects/jaffle-shop-data.zip"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_yaml_file",
"check_dbt_command",
"check_database",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "dbt_profiles",
"paths": [
"/home/user/.dbt/profiles.yml",
"/home/user/projects/jaffle_shop/profiles.yml",
"$DBT_PROFILES_DIR"
],
"dest": "profiles.yml"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"]
},
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"dest": "jaffle_shop.duckdb"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"]
}
],
"expected": [
{
"type": "rule",
"rules": [
["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"],
["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"],
["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]],
["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]]
]
},
{
"type": "rule",
"rules": [
["contains", "OK connection ok", ""],
["contains", "All checks passed", ""]
]
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476",
"dest": "gold_jaffle_shop.duckdb"
},
{
"type": "rule",
"rules": [
["excludes", "Nothing to do", ""],
["contains", "Completed successfully", ""],
["contains", "PASS=3", ""],
["contains", "TOTAL=3", ""]
]
}
],
"options": [
{},
{},
{
"db_type": "duckdb",
"check_type": ["table-schema-content"]
},
{}
]
}
}

86
evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json Normal file
View File

@@ -0,0 +1,86 @@
{
"id": "8ff98608-8e0e-526e-9413-d744554ba708",
"snapshot": "dbt",
"instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=10",
"https://docs.getdbt.com/guides/manual-install?step=8",
"https://docs.getdbt.com/guides/manual-install?step=9"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_database",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"dest": "jaffle_shop.duckdb"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"]
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749",
"dest": "gold_jaffle_shop.duckdb"
},
{
"type": "rule",
"rules": [
["contains", "1 of 1 OK created sql table model .*\\.customers", ""],
["contains", "Completed successfully", ""],
["contains", "PASS=1", ""],
["contains", "TOTAL=1", ""]
]
}
],
"options": [
{
"db_type": "duckdb",
"check_type": ["table-schema-content"]
},
{}
]
}
}

136
evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json Normal file
View File

@@ -0,0 +1,136 @@
{
"id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
"snapshot": "chrome",
"instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'figures' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.freerice.com/",
"https://www.hku.hk/",
"https://about.meta.com/technologies/facebook-app/"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "compare_figures",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"query_list": [
[
"title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"title = '1.png' and trashed = false"
],
[
"title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"title = '2.png' and trashed = false"
],
[
"title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"title = '3.png' and trashed = false"
]
],
"dest": [
"1.png",
"2.png",
"3.png"
]
},
"expected": {
"type": "cloud_file",
"path": [
"file1",
"file2",
"file3"
],
"dest": [
"1_gold.png",
"2_gold.png",
"3_gold.png"
],
"multi": true,
"gives": [0, 1, 2]
}
}
}

111
evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json Normal file
View File

@@ -0,0 +1,111 @@
{
"id": "b52b40a5-ad70-4c53-b5b0-5650a8387052",
"snapshot": "chrome",
"instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'attachment_full.pdf' and 'root' in parents",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.freerice.com/",
"https://www.hku.hk/",
"https://about.meta.com/technologies/facebook-app/"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "compare_pdfs",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"query": [
"title = 'attachment_full.pdf' and trashed = false and 'root' in parents"
],
"dest": "attachment_full.pdf"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805",
"dest": "attachment_full_gold.pdf"
}
}
}

2
evaluation_examples/settings/google/settings.json
View File

@@ -1,4 +1,4 @@
{
"account": "xlang2024anonym@gmail.com",
"email": "xlang2024anonym@gmail.com",
"password": "q]wN~0iD>H:6"
}

2
evaluation_examples/settings/googledrive/credentials.json
View File

@@ -1 +1 @@
{"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T05:51:17Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
{"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T16:47:34Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}

5
evaluation_examples/settings/thunderbird/settings.json Normal file
View File

@@ -0,0 +1,5 @@
{
"email": "anonym-x2024@outlook.com",
"password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>",
"user": "Anonym Tester"
}