lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2024-01-26 16:14:51 +08:00
parent 78c7b702b3 35cc698ffa
commit 379f1f5a6c
39 changed files with 1506 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/controllers/setup.py
View File

@@ -108,7 +108,6 @@ class SetupController:
cache_path: str = os.path.join(self.cache_dir, "{:}_{:}".format(
uuid.uuid5(uuid.NAMESPACE_URL, url),
os.path.basename(path)))
if not url or not path:
raise Exception(f"Setup Download - Invalid URL ({url}) or path ({path}).")
@@ -536,7 +535,7 @@ class SetupController:
page.goto(url)
logger.info(f"Opened new page: {url}")
settings = json.load(open(config['settings_file']))
email, password = settings['account'], settings['password']
email, password = settings['email'], settings['password']
try:
page.wait_for_selector('input[type="email"]', state="visible", timeout=3000)

6
desktop_env/evaluators/getters/__init__.py
View File

@@ -4,8 +4,10 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g
from .file import get_cloud_file, get_vm_file, get_cache_file
from .general import get_vm_command_line
from .impress import get_audio_in_slide
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory
from .misc import get_rule, get_accessibility_tree
from .replay import get_replay
from .vlc import get_vlc_playing_info, get_vlc_config
from .vscode import get_vscode_config
from .vscode import get_vscode_config
from .impress import get_audio_in_slide

65
desktop_env/evaluators/getters/chrome.py
View File

@@ -469,34 +469,51 @@ def get_googledrive_file(env, config: Dict[str, Any]) -> str:
To retrieve the file, we provide two options in config dict:
1. query: a list of queries to search the file, each query is a string that follows the format of Google Drive search query
2. path: a list of path to the file, 'folder/subfolder/filename' -> ['folder', 'subfolder', 'filename']
3. query_list: query extends to list to download multiple files
4. path_list: path extends to list to download multiple files
dest: target file name or list. If *_list is used, dest should also be a list of the same length.
Return the downloaded filepath locally.
"""
settings_file = config.get('settings_file', 'evaluation_examples/settings/googledrive/settings.json')
auth = GoogleAuth(settings_file=settings_file)
drive = GoogleDrive(auth)
_path = os.path.join(env.cache_dir, config['dest'])
def get_single_file(_query, _path):
parent_id = 'root'
try:
for q in _query:
search = f'( {q} ) and "{parent_id}" in parents'
filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList()
if len(filelist) == 0: # target file not found
return None
file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one
parent_id = file['id']
file.GetContentFile(_path, mimetype=file['mimeType'])
except:
logger.info('[ERROR]: Failed to download the file from Google Drive')
return None
return _path
if 'query' in config:
query = config['query']
if type(query) != list: query = [query]
else:
paths = config['path']
if type(paths) != list: paths = [paths]
query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(paths) - 1
else f'title = {fp} and trashed = false' for idx, fp in enumerate(paths)]
parent_id = 'root'
try:
for q in query:
search = f'( {q} ) and "{parent_id}" in parents'
filelist: GoogleDriveFileList = drive.ListFile({'q': search}).GetList()
if len(filelist) == 0: # target file not found
return None
file: GoogleDriveFile = filelist[0] # HACK: if multiple candidates, just use the first one
parent_id = file['id']
file.GetContentFile(_path, mimetype=file['mimeType'])
except:
logger.info('[ERROR]: Failed to download the file from Google Drive')
return None
return _path
return get_single_file(config['query'], os.path.join(env.cache_dir, config['dest']))
elif 'path' in config:
query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if idx < len(config['path']) - 1
else f'title = {fp} and trashed = false' for idx, fp in enumerate(config['path'])]
return get_single_file(query, os.path.join(env.cache_dir, config['dest']))
elif 'query_list' in config:
_path_list = []
assert len(config['query_list']) == len(config['dest'])
for idx, query in enumerate(config['query_list']):
dest = config['dest'][idx]
_path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest)))
return _path_list
else: # path_list in config
_path_list = []
assert len(config['path_list']) == len(config['dest'])
for idx, path in enumerate(config['path_list']):
query = [f"title = {fp} and mimeType = 'application/vnd.google-apps.folder' and trashed = false" if jdx < len(path) - 1
else f'title = {fp} and trashed = false' for jdx, fp in enumerate(path)]
dest = config['dest'][idx]
_path_list.append(get_single_file(query, os.path.join(env.cache_dir, dest)))
return _path_list

4
desktop_env/evaluators/getters/info.py
View File

@@ -18,3 +18,7 @@ def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]:
f.write(content)
return _path
def get_list_directory(env, config: dict) -> dict:
return env.controller.get_vm_directory_tree(config["path"])

11
desktop_env/evaluators/metrics/__init__.py
View File

@@ -4,7 +4,9 @@ from .chrome import is_expected_tabs, is_expected_bookmarks, compare_pdfs, is_co
from .docs import compare_font_names, compare_subscript_contains, has_page_numbers_in_footers, compare_docx_lines
from .docs import evaluate_colored_words_in_tables, check_highlighted_words, evaluate_strike_through_last_paragraph, \
evaluate_conversion, evaluate_spacing, check_italic_font_size_14, evaluate_alignment, get_unique_train_ids, \
check_no_duplicates
check_no_duplicates, compare_init_lines
from .general import exact_match, fuzzy_match
from .docs import find_default_font, contains_page_break, compare_docx_files, compare_docx_tables, compare_line_spacing, \
compare_insert_equation, compare_highlighted_text
from .docs import is_first_line_centered, check_file_exists, compare_contains_image
@@ -19,10 +21,15 @@ from .slides import check_presenter_console_disable, check_image_stretch_and_cen
# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \
check_qt_max_volume, \
check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
check_json_keybindings
from .os import check_gnome_favorite_apps
from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs

47
desktop_env/evaluators/metrics/docs.py
View File

@@ -56,15 +56,34 @@ def compare_docx_files(file1, file2):
doc2_paragraphs = [p.text for p in doc2.paragraphs]
if len(doc1_paragraphs) != len(doc2_paragraphs):
# print(len(doc1_paragraphs))
# print(len(doc2_paragraphs))
return 0
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if p1 != p2:
# print(p1)
# print(p2)
return 0
return 1
def compare_init_lines(file1, file2):
doc1 = Document(file1)
doc2 = Document(file2)
doc1_paragraphs = [p.text for p in doc1.paragraphs]
doc2_paragraphs = [p.text for p in doc2.paragraphs]
# Compare each paragraph
for p1, p2 in zip(doc1_paragraphs, doc2_paragraphs):
if p1 != p2:
# print(p1)
# print(p2)
return 0
return 1
def compare_docx_tables(docx_file1, docx_file2):
doc1 = Document(docx_file1)
@@ -93,6 +112,8 @@ def compare_docx_tables(docx_file1, docx_file2):
def compare_line_spacing(docx_file1, docx_file2):
if not compare_docx_files(docx_file1, docx_file2):
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -200,8 +221,10 @@ def compare_contains_image(docx_file1, docx_file2):
# print(find_default_font("Ani", config_path))
def evaluate_colored_words_in_tables(file_path):
document = Document(file_path)
def evaluate_colored_words_in_tables(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
for table in document.tables:
# Iterate through rows and cells in the table
@@ -221,8 +244,10 @@ def evaluate_colored_words_in_tables(file_path):
return 1 # All words in tables are correctly colored
def check_highlighted_words(file_path):
document = Document(file_path)
def check_highlighted_words(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
for paragraph in document.paragraphs:
for run in paragraph.runs:
@@ -232,8 +257,10 @@ def check_highlighted_words(file_path):
return 1 # No highlighted words found
def evaluate_strike_through_last_paragraph(file_path):
document = Document(file_path)
def evaluate_strike_through_last_paragraph(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
document = Document(file_path1)
# Get the last paragraph
last_paragraph = document.paragraphs[-1]
@@ -278,8 +305,10 @@ def evaluate_spacing(file_path):
return 0
def check_italic_font_size_14(path):
document = Document(path)
def check_italic_font_size_14(path1, path2):
if not compare_docx_files(path1, path2):
return 0
document = Document(path1)
for paragraph in document.paragraphs:
for run in paragraph.runs:
if run.italic:
@@ -365,6 +394,8 @@ def compare_docx_lines(file1, file2):
doc2 = Document(file2)
doc2_lines = [p.text.strip() for p in doc2.paragraphs if p.text.strip()]
# print(doc1_lines)
# print(doc2_lines)
# Convert the list of lines to sets and compare
return set(doc1_lines) == set(doc2_lines)

77
desktop_env/evaluators/metrics/os.py
View File

@@ -17,54 +17,43 @@ def check_gnome_favorite_apps(apps_str: str, rule):
return 0
# TODO: log in to the system before running this
def is_logout_successful():
try:
subprocess.run(["whoami"])
return 0 # Task not successful
except subprocess.CalledProcessError:
return 1 # Task successful
def is_utc_0(timedatectl_output):
"""
Format as:
Local time: Thu 2024-01-25 12:56:06 WET
Universal time: Thu 2024-01-25 12:56:06 UTC
RTC time: Thu 2024-01-25 12:56:05
Time zone: Atlantic/Faroe (WET, +0000)
System clock synchronized: yes
NTP service: inactive
RTC in local TZ: no
"""
utc_line = timedatectl_output.split("\n")[3]
if utc_line.endswith("+0000)"):
return 1
else:
return 0
def is_battery_percentage_displayed():
# GNOME schema and key for the setting
schema = "org.gnome.desktop.interface"
key = "show-battery-percentage"
try:
# use gsettings to get the current setting
result = subprocess.run(['gsettings', 'get', schema, key], capture_output=True, text=True)
# examine the output is 'true'
if result.stdout.strip() == 'true':
return 1.
else:
return 0.
except Exception as e:
print(f"An error occurred: {e}")
return 0.
def check_text_enlarged(scaling_factor_str):
scaling_factor = float(scaling_factor_str)
if scaling_factor > 1.0:
return 1
else:
return 0
def check_auto_lock_settings():
# The schema and keys for the GNOME desktop lock settings
schema = "org.gnome.desktop.screensaver"
lock_enabled_key = "lock-enabled"
lock_delay_key = "lock-delay"
def check_moved_jpgs(directory_list, rule):
try:
# Check if the screen lock is enabled
lock_enabled = subprocess.run(['gsettings', 'get', schema, lock_enabled_key], capture_output=True, text=True)
is_lock_enabled = lock_enabled.stdout.strip() == 'true'
expected_jpgs = rule["expected"]
moved_jpgs = [node['name'] for node in directory_list['children']]
# Check the delay before the screen is locked
lock_delay = subprocess.run(['gsettings', 'get', schema, lock_delay_key], capture_output=True, text=True)
# Extract the numerical value from the output
delay_seconds = lock_delay
if len(moved_jpgs) != len(expected_jpgs):
return 0
# Print the results
if is_lock_enabled:
return 1.
else:
return 0.
except Exception as e:
return 0.
if set(moved_jpgs) == set(expected_jpgs):
return 1
else:
return 0

119
evaluation_examples/examples/dbt/0aa56709-3293-5849-ad47-e377f49fd3a0.json Normal file
View File

@@ -0,0 +1,119 @@
{
"id": "0aa56709-3293-5849-ad47-e377f49fd3a0",
"snapshot": "dbt",
"instruction": "Update models/schema.yml file to include some description fields. 1) The description fields include what each table is about. 2) For the primary key column of each table, add the description \"Primary key\". 3) Also insert one description for column customers.first_order_date that \"NULL when a customer has not yet placed an order.\". Then, use dbt docs command to generate the documentation for this dbt project and launch the documentation in a local website with port 8020.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=13"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1TZmmW7wYnWUQVMCH5JOBCxxdgm_QN-vz&export=download&authuser=0&confirm=t&uuid=43113538-c0e4-4e23-8a18-12b727b9f890&at=APZUnTWgSRXjNNZZIt8ni7rsTxoy:1705910001969",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_yaml_file",
"check_dbt_command",
"check_dbt_command",
"check_dbt_command",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/models/schema.yml",
"dest": "schema.yml"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; if [ -s target/catalog.json ] && [ -s target/index.html ] ; then echo \"dbt docs generate succeed\" ; else echo \"dbt docs generate failed\" ; fi"]
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; curl -o index.html http://localhost:8020; if [ -s index.html ] ; then echo \"dbt docs serve succeed\" ; else echo \"dbt docs serve failed\" ; fi"]
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; diff index.html target/index.html > diff.log; if [ -s diff.log ] ; then echo \"dbt docs serve failed\" ; else echo \"dbt docs serve succeed\" ; fi ; rm -rf diff.log index.html"]
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt docs generate"]
}
],
"expected": [
{
"type": "rule",
"rules": [
["not_null", ["models", ["name", "customers"], "description"], ""],
["not_null", ["models", ["name", "stg_customers"], "description"], ""],
["not_null", ["models", ["name", "stg_orders"], "description"], ""],
["match", ["models", ["name", "customers"], "columns", ["name", "customer_id"], "description"], "Primary key"],
["match", ["models", ["name", "stg_customers"], "columns", ["name", "customer_id"], "description"], "Primary key"],
["match", ["models", ["name", "stg_orders"], "columns", ["name", "order_id"], "description"], "Primary key"],
["match", ["models", ["name", "customers"], "columns", ["name", "first_order_date"], "description"], "NULL when a customer has not yet placed an order."]
]
},
{
"type": "rule",
"rules": [
["contains", "dbt docs generate succeed", ""],
["excludes", "dbt docs generate failed", ""]
]
},
{
"type": "rule",
"rules": [
["contains", "dbt docs serve succeed", ""],
["excludes", "dbt docs serve failed", ""]
]
},
{
"type": "rule",
"rules": [
["contains", "dbt docs serve succeed", ""],
["excludes", "dbt docs serve failed", ""]
]
},
{
"type": "rule",
"rules": [
["excludes", "error", ""],
["excludes", "Error", ""],
["contains", "Catalog written to", ""]
]
}
]
}
}

89
evaluation_examples/examples/dbt/492c2c87-b04a-544a-b5dd-eb808036bf85.json Normal file
View File

@@ -0,0 +1,89 @@
{
"id": "492c2c87-b04a-544a-b5dd-eb808036bf85",
"snapshot": "dbt",
"instruction": "Separate the single dbt model customers into separate ones and use the ``ref`` function to build models on top of others. 1) Create a new SQL file, models/stg_customers.sql, with the SQL from the customers common table expressions (CTE) in the original query. 2) Create a second new SQL file, models/stg_orders.sql, with the SQL from the orders CTE in the original query. These two stg models are materialized as view. 3) Edit the SQL in models/customers.sql file to refer two these two staged models and execute dbt run. Please ensure the running succeeds.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=11"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-Nn0leqKdVnA1gQ6s50wkGSa2xZauEUe&export=download&authuser=0&confirm=t&uuid=d7596b15-ae39-43cb-a940-03bee8a0c961&at=APZUnTXwp5xMjHpi5AYVF3Z_bfw2:1705582080467",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_database",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"dest": "jaffle_shop.duckdb"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"]
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1rtRXtgKiwyWSVPCbDVkjYXh8AMni7t1W&export=download&authuser=0&confirm=t&uuid=6e4331f7-4e46-4894-910f-9af77450534d&at=APZUnTXySypK2OcEdOb8FVHQnbd7:1705583620817",
"dest": "gold_jaffle_shop.duckdb"
},
{
"type": "rule",
"rules": [
["contains", "of 3 OK created sql view model .*\\.stg_customers", ""],
["contains", "of 3 OK created sql view model .*\\.stg_orders", ""],
["contains", "3 of 3 OK created sql table model .*\\.customers", ""],
["contains", "Completed successfully", ""],
["contains", "PASS=3", ""],
["contains", "TOTAL=3", ""]
]
}
],
"options": [
{
"db_type": "duckdb",
"check_type": [
"table-schema-content",
"view-schema-content"
]
},
{}
]
}
}

126
evaluation_examples/examples/dbt/49f981ee-f793-5e27-9a53-083d66934ea1.json Normal file
View File

@@ -0,0 +1,126 @@
{
"id": "49f981ee-f793-5e27-9a53-083d66934ea1",
"snapshot": "dbt",
"instruction": "Add dbt test for the three models in jaffle_shop project. 1) Create a YAML file in the models directory, named models/schema.yml. 2) Ensure that customer_id and order_id are unique and not empty in related tables or views. Column status must be value from placed, shipped, completed, return_pending, returned. And customer_id in stg_orders is not null and is a foreign key in the referenced stg_customers. 3) Run dbt test, and confirm that all tests passed.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=12"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1QhPSdctnfYk0O5Tuo2NzldMDw0cKZi16&export=download&authuser=0&confirm=t&uuid=d68dcd34-304d-4e66-a215-99abd3954c24&at=APZUnTXihMvZPodIOVVxg3S0tUs4:1705587177507",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": "check_dbt_command",
"result": {
"type": "dbt_test_result",
"pre-processing": ["mv", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb", "/home/user/Desktop/jaffle_shop.duckdb"],
"path": [
"https://drive.usercontent.google.com/download?id=1AZ6hCtbyN8Ypzf0e2nkkxivxdAmKNHZQ&export=download&authuser=0&confirm=t&uuid=dffccf1e-a42b-45e7-966d-72d289a1062e&at=APZUnTUsGPahiUmvgXxji9x9Ii7o:1705668446168",
"https://drive.usercontent.google.com/download?id=1z0hApNSqvs2oUwJiBQmFXrsomAxa1RhJ&export=download&authuser=0&confirm=t&uuid=0a2ba1be-1e15-4e8d-8458-0a661eaeef6f&at=APZUnTVZMe3y1OpU6ipsm0U6Ryb6:1705668535249",
"https://drive.usercontent.google.com/download?id=1OYsLQSYAdaAyu0sa6Y8IstmWz7wXneUN&export=download&authuser=0&confirm=t&uuid=33818f3c-a125-44d4-b9ba-7c5465976250&at=APZUnTXWFrNUtydf460ZWA-2jJrg:1705668651799",
"https://drive.usercontent.google.com/download?id=1KgGuJMeCXpIG2_TAKIJsT-YoIbSWswYp&export=download&authuser=0&confirm=t&uuid=4f2aadaa-2bc0-4ecf-b725-19c73375e370&at=APZUnTVt5cFNMzCsRxekJUaBPfhw:1705668748132",
"https://drive.usercontent.google.com/download?id=1OJ0xgAF1KqhovIvkimp3J5ZM-crfZU6g&export=download&authuser=0&confirm=t&uuid=48bcfac9-d40c-40c5-ac07-da9288a4ebea&at=APZUnTVUl1FaKtXy4oVpkYnGKIdt:1705669212860",
"https://drive.usercontent.google.com/download?id=10mr1jhdA52_bKOgoDgG0mAal1nzXkG21&export=download&authuser=0&confirm=t&uuid=5adde4ce-59d5-4da0-ada8-9c7df3e7434c&at=APZUnTVwJ4WDHSnfUv8yB08jr1b7:1705669280449",
"https://drive.usercontent.google.com/download?id=1_Dh9lwVDo8TfB0jg8QsdemRZ14r9O-uY&export=download&authuser=0&confirm=t&uuid=a8445889-8d13-4b66-88e7-4b675a943bec&at=APZUnTWrfFiAMJVLNFuteOctpjzs:1705669328524",
"https://drive.usercontent.google.com/download?id=1n4fLzFWj9dEqRdLOSwSB5SB8O3kHyEBZ&export=download&authuser=0&confirm=t&uuid=78cd4358-cc9e-4d2c-b74f-93b7760de4d3&at=APZUnTXbEj7FEQL5DEU5g4wsDFHs:1705669396969",
"https://drive.usercontent.google.com/download?id=1sjwB1aoi5UzbHQI-mb0hph2qd4qRKZR_&export=download&authuser=0&confirm=t&uuid=2fe93a52-d820-462e-9cce-36daa3a5ce30&at=APZUnTXPsOgN81L1RkenHg__oXUB:1705669474430",
"https://drive.usercontent.google.com/download?id=1ujsyfS7F3A6YCgZJ7N9ixcRlgNdAAdXJ&export=download&authuser=0&confirm=t&uuid=7e1b73c9-a14b-4d8b-800c-77f16ac5f897&at=APZUnTWBVbznnKO-zYj7UI6qk-qz:1705669501449",
"https://drive.usercontent.google.com/download?id=1tujswM-r4GKav5CpmciF-H3zYAdEq-uL&export=download&authuser=0&confirm=t&uuid=32253ad7-d343-4961-bc14-1e4cf4fee244&at=APZUnTWiRpT1pHF6Qp17y1VqcT7Z:1705676183100"
],
"dest": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt test"],
"post-processing": ["/bin/bash", "-c", "mv /home/user/Desktop/jaffle_shop.duckdb /home/user/projects/jaffle_shop/jaffle_shop.duckdb"]
},
"expected": {
"type": "rule",
"rules": [
[
["excludes", "Nothing to do", ""],
["contains", "Completed successfully", ""],
["contains", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["excludes", "Completed successfully", ""],
["excludes", "ERROR=0", ""]
],
[
["excludes", "Nothing to do", ""],
["contains", "Completed successfully", ""],
["contains", "ERROR=0", ""]
]
]
}
}
}

102
evaluation_examples/examples/dbt/8aa9e870-b0c9-5417-be80-03154e83c7a3.json Normal file
View File

@@ -0,0 +1,102 @@
{
"id": "8aa9e870-b0c9-5417-be80-03154e83c7a3",
"snapshot": "dbt",
"instruction": "Create a dbt project called jaffle_shop under ~/projects and configure the connection to duckdb with path jaffle_shop/jaffle_shop.duckdb for the target dev and prod. Load data from jaffle-shop-data.zip into that database using dbt.",
"source": [
"https://docs.getdbt.com/docs/core/connect-data-platform/duckdb-setup#",
"https://docs.getdbt.com/guides/manual-install?step=3",
"https://docs.getdbt.com/reference/commands/seed"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=18pbzQOnAZEZ2psFLWJJQIyLlQBSNRxWs&export=download&authuser=0&confirm=t&uuid=d8045faa-0f74-4c34-ab98-55ab41bcc2c9&at=APZUnTUoi3u-jMpwO8osMEll4gDr:1705548224569",
"path": "/home/user/projects/jaffle-shop-data.zip"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_yaml_file",
"check_dbt_command",
"check_database",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "dbt_profiles",
"paths": [
"/home/user/.dbt/profiles.yml",
"/home/user/projects/jaffle_shop/profiles.yml",
"$DBT_PROFILES_DIR"
],
"dest": "profiles.yml"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt debug"]
},
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"dest": "jaffle_shop.duckdb"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt seed"]
}
],
"expected": [
{
"type": "rule",
"rules": [
["match", ["jaffle_shop", "outputs", "dev", "type"], "duckdb"],
["match", ["jaffle_shop", "outputs", "prod", "type"], "duckdb"],
["in", ["jaffle_shop", "outputs", "dev", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]],
["in", ["jaffle_shop", "outputs", "prod", "path"], ["jaffle_shop.duckdb", "/home/user/projects/jaffle_shop/jaffle_shop.duckdb"]]
]
},
{
"type": "rule",
"rules": [
["contains", "OK connection ok", ""],
["contains", "All checks passed", ""]
]
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=12xJuEcBxGqPHjJriUPnkuGRkw9HsZniJ&export=download&authuser=0&confirm=t&uuid=f2b7147f-5114-461d-a939-486750c19029&at=APZUnTVBf4QJMyDPk5BmfZzTecU5:1705548962476",
"dest": "gold_jaffle_shop.duckdb"
},
{
"type": "rule",
"rules": [
["excludes", "Nothing to do", ""],
["contains", "Completed successfully", ""],
["contains", "PASS=3", ""],
["contains", "TOTAL=3", ""]
]
}
],
"options": [
{},
{},
{
"db_type": "duckdb",
"check_type": ["table-schema-content"]
},
{}
]
}
}

86
evaluation_examples/examples/dbt/8ff98608-8e0e-526e-9413-d744554ba708.json Normal file
View File

@@ -0,0 +1,86 @@
{
"id": "8ff98608-8e0e-526e-9413-d744554ba708",
"snapshot": "dbt",
"instruction": "Delete the dbt example models and write one dbt model \"customers\" from existing tables raw_customers and raw_orders. It contains columns customer_ids, first_name, last_name, first_order_date, most_recent_order_date and number_of_orders (default to 0). Set the materialization method as table and build it with dbt run to make it work.",
"source": [
"https://docs.getdbt.com/guides/manual-install?step=10",
"https://docs.getdbt.com/guides/manual-install?step=8",
"https://docs.getdbt.com/guides/manual-install?step=9"
],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1iqimXoJUa2fd16I_qCZGHbVMHABJAh7D&export=download&authuser=0&confirm=t&uuid=65b73e29-0b30-415d-9928-d1ccf3be9880&at=APZUnTWCBNMro4gX8whSahDtx05S:1705563480661",
"path": "/home/user/projects/jaffle_shop.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["/bin/bash", "-c", "unzip -oq /home/user/projects/jaffle_shop.zip -d /home/user/projects/ && rm /home/user/projects/jaffle_shop.zip && mkdir -p /home/user/.dbt"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1xkXjiFhRNoiX_-0zikfdwzJ8UvdXEVAt&export=download&authuser=0&confirm=t&uuid=bc8d0eb8-99b3-4be7-b110-c0a56246d2d2&at=APZUnTUaPD_Z6ov6uMWDNf5rSy3-:1705978221799",
"path": "/home/user/.dbt/profiles.yml"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"dbt",
"duckdb"
],
"evaluator": {
"func": [
"check_database",
"check_dbt_command"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/projects/jaffle_shop/jaffle_shop.duckdb",
"dest": "jaffle_shop.duckdb"
},
{
"type": "vm_command_line",
"command": ["/bin/bash", "-c", "cd /home/user/projects/jaffle_shop; source ~/anaconda3/etc/profile.d/conda.sh && conda activate dbt; dbt run"]
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1BnTQOiDIKriWGhLRj7C8MPgfk4JH-uf9&export=download&authuser=0&confirm=t&uuid=cee74971-d66b-4da0-8cdd-cb9f1b9da8e7&at=APZUnTWI7GqSBLnQ_g-NNsS7kDxq:1705567576749",
"dest": "gold_jaffle_shop.duckdb"
},
{
"type": "rule",
"rules": [
["contains", "1 of 1 OK created sql table model .*\\.customers", ""],
["contains", "Completed successfully", ""],
["contains", "PASS=1", ""],
["contains", "TOTAL=1", ""]
]
}
],
"options": [
{
"db_type": "duckdb",
"check_type": ["table-schema-content"]
},
{}
]
}
}

7
evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
View File

@@ -19,8 +19,9 @@
"type": "open",
"parameters": {
"path": "/home/user/OrderId_Month_Chart.xlsx"
]
},
}
}
],
"trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
"related_apps": [
"libreoffice calc"
@@ -77,4 +78,4 @@
]
}
}
}
}

79
evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json Normal file
View File

@@ -0,0 +1,79 @@
{
"id": "39aa4e37-dc91-482e-99af-132a612d40f3",
"snapshot": "libreoffice_calc",
"instruction": "Help me rename sheet1 \"LARS_Science_Assessment\"",
"source": "https://www.libreofficehelp.com/add-insert-delete-copy-move-rename-a-worksheet-in-libreoffice-calc/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Y9KUHsACajMU1kWz_XjW0VBO3RleDBXw&export=download&authuser=0&confirm=t&uuid=73fe859e-7fe1-449b-83cd-2ddaab067f90&at=APZUnTW-Gp9kn8VhJVCXi5VWxjMT:1706099595303",
"path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx"
}
}
],
"trajectory": "trajectories/39aa4e37-dc91-482e-99af-132a612d40f3",
"related_apps": [
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/LARS_Science_Assessment_Resource_List_7-30-21.xlsx",
"dest": "LARS_Science_Assessment_Resource_List_7-30-21.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1JC8iKt5_XjhksBDSePi0THwrfM5oF-5A&export=download&authuser=0&confirm=t&uuid=6124298b-1487-4a85-b7d9-d9ba133c26e0&at=APZUnTVfW7o4XruPUft-Dph7um6J:1706099593830",
"dest": "LARS_Science_Assessment_Resource_List_7-30-21_gold.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_name"
},
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

0
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup → evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json
View File

27
evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json
View File

@@ -27,7 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": "compare_docx_files",
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "04 CHIN9505 EBook Purchasing info 2021 Jan.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_init_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1yyHGj8KUHDMsZmc1QeJ1KkvSEGy83jMR&export=download",

30
evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=df2bb3c3-e75e-4cbc-a6bc-24120512a4e1&at=APZUnTUak54ZfgQDNxbt_PqBvNQu:1706017722797",
"url": "https://drive.usercontent.google.com/download?id=10hgB73d_DoQXQVgUjvgXFUCP1Hd9YxDb&export=download&authuser=0&confirm=t&uuid=845f9616-2fb7-476a-abab-8b620d482ac2&at=APZUnTXB71PxHF7Dq9TC2OL_cRLm:1706199789147",
"path": "Desktop/sample-recruitment-phone-script.docx"
}
]
@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"compare_docx_files",
"check_highlighted_words"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "sample-recruitment-phone-script.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "check_highlighted_words",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1s9Dsy66-zxbCAgeTyCh0P7AT7P4jF6o3&export=download&authuser=0&confirm=t&uuid=1239f2a1-8c86-45a4-8e7d-36388ac22a69&at=APZUnTVZQzXQAMNsKKQzOw5ppT8A:1706017721589",

25
evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json
View File

@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "HK_train_record.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_lines",
"result": {
"type": "vm_file",

30
evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "GEOG2169_Course_Outline_2022-23.docx"
"path": "Desktop/GEOG2169_Course_Outline_2022-23.docx"
}
}
],
@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"evaluate_strike_through_last_paragraph",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "GEOG2169_Course_Outline_2022-23.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "evaluate_strike_through_last_paragraph",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1IpAnQRYo1whrnzIGyo8UldZf4Tli-yVT&export=download",

28
evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json
View File

@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"evaluate_colored_words_in_tables",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Dolch_Sight_Words_Primer.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "evaluate_colored_words_in_tables",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1ksn444K17lFOdm5pELrQYvuZHkOsKq69&export=download",

27
evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "CCCH9003_Tutorial_guidelines.docx"
"path": "Desktop/CCCH9003_Tutorial_guidelines.docx"
}
}
],
@@ -27,6 +27,31 @@
"libreoffice_writer"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "CCCH9003_Tutorial_guidelines.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",

30
evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "CCHU9045_Course_Outline_2019-20.docx"
"path": "Desktop/CCHU9045_Course_Outline_2019-20.docx"
}
}
],
@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"compare_line_spacing",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "CCHU9045_Course_Outline_2019-20.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "compare_line_spacing",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=16LN7uYSSXk_xwgc4IZXnN2Z1nCmPJfLm&export=download",

27
evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json
View File

@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "presentation_instruction_2023_Feb.docx"
"path": "Desktop/presentation_instruction_2023_Feb.docx"
}
}
],
@@ -28,6 +28,31 @@
],
"evaluator": {
"func": "compare_docx_files",
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "presentation_instruction_2023_Feb.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1bB1N2TWN0puZ6DwUFS_TDjvRWchaGP9T&export=download",

28
evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json
View File

@@ -27,10 +27,32 @@
"libreoffice_writer"
],
"evaluator": {
"func": [
"check_italic_font_size_14",
"compare_docx_files"
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Y22-2119-assign4.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
]
}
}
],
"func": "check_italic_font_size_14",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1GTZ-DkMxpdYx38z_s0ab85Ejgxv3qfEp&export=download",

136
evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json Normal file
View File

@@ -0,0 +1,136 @@
{
"id": "46407397-a7d5-4c6b-92c6-dbe038b1457b",
"snapshot": "chrome",
"instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload them in the figures/ folder in Google Drive for later use (use pure numbers to name them).",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'figures' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.freerice.com/",
"https://www.hku.hk/",
"https://about.meta.com/technologies/facebook-app/"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "compare_figures",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"query_list": [
[
"title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"title = '1.png' and trashed = false"
],
[
"title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"title = '2.png' and trashed = false"
],
[
"title = 'figures' and trashed = false and 'root' in parents and mimeType = 'application/vnd.google-apps.folder'",
"title = '3.png' and trashed = false"
]
],
"dest": [
"1.png",
"2.png",
"3.png"
]
},
"expected": {
"type": "cloud_file",
"path": [
"file1",
"file2",
"file3"
],
"dest": [
"1_gold.png",
"2_gold.png",
"3_gold.png"
],
"multi": true,
"gives": [0, 1, 2]
}
}
}

111
evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json Normal file
View File

@@ -0,0 +1,111 @@
{
"id": "b52b40a5-ad70-4c53-b5b0-5650a8387052",
"snapshot": "chrome",
"instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'attachment_full.pdf' and 'root' in parents",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.freerice.com/",
"https://www.hku.hk/",
"https://about.meta.com/technologies/facebook-app/"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "compare_pdfs",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"query": [
"title = 'attachment_full.pdf' and trashed = false and 'root' in parents"
],
"dest": "attachment_full.pdf"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805",
"dest": "attachment_full_gold.pdf"
}
}
}

112
evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json
View File

@@ -1,17 +1,101 @@
{
"id": "23393935-50c7-4a86-aeea-2b78fd089c5c",
"snapshot": "os",
"instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.",
"source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "23393935-50c7-4a86-aeea-2b78fd089c5c",
"snapshot": "os",
"instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.",
"source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": "mkdir -p ~/Desktop/photos && mkdir -p ~/Desktop/cpjpg",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mkdir -p ~/Desktop/photos/vacation && mkdir -p ~/Desktop/photos/family && mkdir -p ~/Desktop/photos/events",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mkdir -p ~/Desktop/photos/vacation/thailand && mkdir -p ~/Desktop/photos/vacation/hk",
"shell": true
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1H7R6flpv6vgyBOHurzR0CyvPRIobuIpI&export=download&authuser=0&confirm=t&uuid=1b41e87d-7a84-4aa6-af40-cf940e80d956&at=APZUnTWus2n1pP6l2wmIyXkCUHGs:1706196492307",
"path": "/home/user/Desktop/photos/vacation/thailand/monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Mh3kIViHaayxNC88TpgpjbmIEl17LA3d&export=download&authuser=0&confirm=t&uuid=a8355b84-aa2e-4502-8d0f-3723ffe7d442&at=APZUnTW4nzDFgDgNeERMRKEQ4bTU:1706196493503",
"path": "/home/user/Desktop/photos/vacation/hk/hong-kong-china.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EhmbZ3QP_mdvvppC5zKA3o--Ce2F-ocO&export=download&authuser=0&confirm=t&uuid=70e05133-dcac-4168-b555-82da15a10d58&at=APZUnTXJ371jG7qiPLHBLt2zy7ij:1706196494612",
"path": "/home/user/Desktop/photos/vacation/hk/hk_group_photo.jpg"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1p_r_BMCLFG872uH9Qa-sLnqXppHvDx95&export=download&authuser=0&confirm=t&uuid=af59d107-8e46-4d2c-b72a-e4b42655959a&at=APZUnTU2hBidSnefJuUQTyJCEVFx:1706196491030",
"path": "/home/user/Desktop/photos/family/us_3.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1YxryNHfLPcFODnUh31CYGHGj3ZFRhvbA&export=download&authuser=0&confirm=t&uuid=0925ae33-ab33-42b9-901c-d90f11e0afcd&at=APZUnTVey7VeC9zYOmTwl0x02Iwf:1706196495770",
"path": "/home/user/Desktop/photos/events/emnlp2023.jpg"
}
]
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "check_moved_jpgs",
"result": {
"type": "list_directory",
"path": "/home/user/Desktop/cpjpg"
},
"expected": {
"type": "rule",
"rules": {
"expected": ["emnlp2023.jpg", "hk_group_photo.jpg", "hong-kong-china.jpg", "monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"]
}
}
}
}

9
evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json
View File

@@ -1,7 +1,7 @@
{
"id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8",
"snapshot": "os",
"instruction": "I want to make the text on the screen larger",
"instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en",
"trajectory": "trajectories/",
"config": [],
@@ -9,10 +9,11 @@
"os"
],
"evaluator": {
"func": "",
"func": "check_text_enlarged",
"result": {
},
"expected": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface text-scaling-factor",
"shell": true
}
}
}

2
evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json
View File

@@ -1,7 +1,7 @@
{
"id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63",
"snapshot": "os",
"instruction": "I want to change my system language to Chinese(simplified). Can you help me?",
"instruction": "I want to change my system language to Chinese (simplified). Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN",
"trajectory": "trajectories/",
"config": [],

40
evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json
View File

@@ -1,18 +1,44 @@
{
"id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
"snapshot": "os",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?",
"instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a poster of party night. Could you help me recover it from the Trash?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1XaTnC_lLbR_tGTz8tcN2Tp6cNrMlNW3R&export=download&authuser=0&confirm=t&uuid=89e69a23-43cf-4316-833a-fb9d3e281460&at=APZUnTWn5zZTH4GlClO6lV1i4WwP:1706184669922",
"path": "poster_party_night.webp"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/",
"shell": true
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -f /home/user/Desktop/poster_party_night.webp ] && echo 'File exists.' || echo 'File does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "File exists.\n"
}
}
}
}

57
evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json
View File

@@ -4,15 +4,60 @@
"instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?",
"source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu",
"trajectory": "trajectories/",
"config": [],
"config": [
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S apt-get install zsh -y",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S chsh -s $(which zsh)",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "echo password | sudo -S loginctl terminate-user user",
"shell": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 10
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(2); pyautogui.press(\"enter\"); pyautogui.type(\"password\"); time.sleep(2); pyautogui.press(\"enter\");"
]
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "echo $SHELL",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "/bin/bash\n"
}
}
}
}

35
evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json
View File

@@ -1,17 +1,24 @@
{
"id": "a4d98375-215b-4a4d-aee9-3d4370fccc41",
"snapshot": "os",
"instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "a4d98375-215b-4a4d-aee9-3d4370fccc41",
"snapshot": "os",
"instruction": "I want to have my computer automatically locked after I leaved. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.screensaver lock-enabled",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"expected": "true\n"
}
}
}
}
}

10
evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json
View File

@@ -4,15 +4,17 @@
"instruction": "I want to set my current time zone to UTC+0. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en",
"trajectory": "trajectories/",
"config": [],
"config": [
],
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"func": "is_utc_0",
"result": {
},
"expected": {
"type": "vm_command_line",
"command": "timedatectl status",
"shell": true
}
}
}

45
evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json
View File

@@ -1,17 +1,36 @@
{
"id": "ddc75b62-7311-4af8-bfb3-859558542b36",
"snapshot": "os",
"instruction": "I want to uninstall the Firefox on my system. Can you help me",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
"id": "ddc75b62-7311-4af8-bfb3-859558542b36",
"snapshot": "os",
"instruction": "I want to uninstall the Mahjongg on my system. Can you help me?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 1
}
}
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "dpkg -l | grep mahjongg",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"include": [
],
"exclude": [
"mahjongg"
]
}
}
}
}
}

44
evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json
View File

@@ -1,17 +1,33 @@
{
"id": "e0df059f-28a6-4169-924f-b9623e7184cc",
"snapshot": "os",
"instruction": "I have a directory named \"test1\". Can you help me change its name into \"test2\"?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en",
"trajectory": "trajectories/",
"related_apps": [
"os"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
"id": "e0df059f-28a6-4169-924f-b9623e7184cc",
"snapshot": "os",
"instruction": "I have a directory named \"todo_list_Jan_1\". Can you help me change its name into \"todo_list_Jan_2\"?",
"source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en",
"trajectory": "trajectories/",
"config": [
{
"type": "execute",
"parameters": {
"command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1",
"shell": true
}
}
}
],
"related_apps": [
"os"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "[ -d ~/Desktop/todo_list_Jan_2 ] && echo 'Directory exists.' || echo 'Directory does not exist.'",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "Directory exists.\n"
}
}
}
}

9
evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json
View File

@@ -8,10 +8,17 @@
"os"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "gsettings get org.gnome.desktop.interface show-battery-percentage",
"shell": true
},
"expected": {
"type": "rule",
"rules":{
"expected": "true\n"
}
}
}
}

2
evaluation_examples/settings/google/settings.json
View File

@@ -1,4 +1,4 @@
{
"account": "xlang2024anonym@gmail.com",
"email": "xlang2024anonym@gmail.com",
"password": "q]wN~0iD>H:6"
}

2
evaluation_examples/settings/googledrive/credentials.json
View File

@@ -1 +1 @@
{"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T05:51:17Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byAHYEB_El-ry3ChNYPLX14Tmq9Nq5j640ghesztCRfoMEPMW79ENOZT8Ab-5wdY7UFxL2ih5l5nhsBf3QENT8W7l7X3QmtQrhyUnDzCfWW2tzYxrNR34iisY0OFWAR9JSfyQPvkpkWQOq0T0QGkBZ8Rg60PVGraNgaCgYKAfQSARISFQHGX2MiTg-uFBVMQnVo8b2H7tuOqw0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}
{"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-25T16:47:34Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byCwgV4hMbm-I9xpB2y3R5GGCfe63GrdDTE22Wv_upDecjO_Ey86lJtENTD_AWVgpB6iR4aidDsq44sJNOBVIMvZtPOfu1Hib3tiyXjS-P4ID0i0NHi21-ZGaewUe1RYBRp_1klpd5KAttaAZC8R-Tn-uSbhKXI0HwaCgYKAaASARISFQHGX2MiD2nlZt98Zb8FQIBcLc7n-A0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"}

5
evaluation_examples/settings/thunderbird/settings.json Normal file
View File

@@ -0,0 +1,5 @@
{
"email": "anonym-x2024@outlook.com",
"password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>",
"user": "Anonym Tester"
}