lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source
This commit is contained in:
Timothyxxx
2024-01-18 01:44:13 +08:00
parent b60eb2a933 d3a9e5088d
commit 2efc6a5b16
37 changed files with 1295 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
desktop_env/controllers/setup.py
View File

@@ -298,11 +298,11 @@ class SetupController:
# TODO
raise NotImplementedError()
def _activate_window_setup(self, window_name: str):
def _activate_window_setup(self, window_name: str, strict: bool = False, by_class: bool = False):
if not window_name:
raise Exception(f"Setup Open - Invalid path ({window_name}).")
payload = json.dumps({"window_name": window_name})
payload = json.dumps({"window_name": window_name, "strict": strict, "by_class": by_class})
headers = {
'Content-Type': 'application/json'
}
@@ -317,6 +317,25 @@ class SetupController:
except requests.exceptions.RequestException as e:
logger.error("An error occurred while trying to send the request: %s", e)
def _close_window_setup(self, window_name: str, strict: bool = False, by_class: bool = False):
if not window_name:
raise Exception(f"Setup Open - Invalid path ({window_name}).")
payload = json.dumps({"window_name": window_name, "strict": strict, "by_class": by_class})
headers = {
'Content-Type': 'application/json'
}
# send request to server to open file
try:
response = requests.post(self.http_server + "/setup" + "/close_window", headers=headers, data=payload)
if response.status_code == 200:
logger.info("Command executed successfully: %s", response.text)
else:
logger.error(f"Failed to close window {window_name}. Status code: %s", response.text)
except requests.exceptions.RequestException as e:
logger.error("An error occurred while trying to send the request: %s", e)
# Chrome setup
def _chrome_open_tabs_setup(self, urls_to_open: List[str]):
host = self.vm_ip

92
desktop_env/evaluators/getters/file.py
View File

@@ -1,50 +1,92 @@
import os
from typing import Dict
from typing import Optional
from typing import Dict, List, Set
from typing import Optional, Any, Union
import requests
def get_cloud_file(env, config: Dict[str, str]) -> str:
def get_cloud_file(env, config: Dict[str, Any]) -> Union[str, List[str]]:
"""
Config:
path (str): the url to download from
dest (str): file name of the downloaded file
path (str|List[str]): the url to download from
dest (str|List[str])): file name of the downloaded file
multi (bool) : optional. if path and dest are lists providing
information of multiple files. defaults to False
gives (List[int]): optional. defaults to [0]. which files are directly
returned to the metric. if len==1, str is returned; else, list is
returned.
"""
_path = os.path.join(env.cache_dir, config["dest"])
if os.path.exists(_path):
return _path
if not config.get("multi", False):
paths: List[str] = [config["path"]]
dests: List[str] = [config["dest"]]
else:
paths: List[str] = config["path"]
dests: List[str] = config["dest"]
cache_paths: List[str] = []
url = config["path"]
response = requests.get(url, stream=True)
response.raise_for_status()
gives: Set[int] = set(config.get("gives", [0]))
with open(_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
for i, (p, d) in enumerate(zip(paths, dests)):
_path = os.path.join(env.cache_dir, d)
if i in gives:
cache_paths.append(_path)
return _path
if os.path.exists(_path):
#return _path
continue
url = p
response = requests.get(url, stream=True)
response.raise_for_status()
with open(_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
return cache_paths[0] if len(cache_paths)==1 else cache_paths
def get_vm_file(env, config: Dict[str, str]) -> Optional[str]:
def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Optional[str]]]:
"""
Config:
path (str): absolute path on the VM to fetch
dest (str): file name of the downloaded file
multi (bool) : optional. if path and dest are lists providing
information of multiple files. defaults to False
gives (List[int]): optional. defaults to [0]. which files are directly
returned to the metric. if len==1, str is returned; else, list is
returned.
"""
_path = os.path.join(env.cache_dir, config["dest"])
if not config.get("multi", False):
paths: List[str] = [config["path"]]
dests: List[str] = [config["dest"]]
else:
paths: List[str] = config["path"]
dests: List[str] = config["dest"]
cache_paths: List[str] = []
file = env.controller.get_file(config["path"])
if file is None:
return None
# raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
with open(_path, "wb") as f:
f.write(file)
gives: Set[int] = set(config.get("gives", [0]))
return _path
for i, (p, d) in enumerate(zip(paths, dests)):
_path = os.path.join(env.cache_dir, d)
file = env.controller.get_file(p)
if file is None:
#return None
# raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
if i in gives:
cache_paths.append(None)
continue
if i in gives:
cache_paths.append(_path)
with open(_path, "wb") as f:
f.write(file)
return cache_paths[0] if len(cache_paths)==1 else cache_paths
def get_cache_file(env, config: Dict[str, str]) -> str:

1
desktop_env/evaluators/metrics/__init__.py
View File

@@ -15,3 +15,4 @@ from .general import check_csv, check_accessibility_tree, check_list, run_sqlite
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed
from .impress import check_slide_numbers_color, compare_pptx_files, check_for_two_lines, check_for_audio, check_formula_shape, check_file_exists
from .impress import check_slide_orientation_Portrait, contains_mp4_video

19
desktop_env/evaluators/metrics/impress.py
View File

@@ -94,6 +94,25 @@ def check_formula_shape(prs):
return 1
return 0
def check_slide_orientation_Portrait(pptx_path):
presentation = Presentation(pptx_path)
slide_height = presentation.slide_height
slide_width = presentation.slide_width
if slide_width < slide_height:
return 1
return 0
def contains_mp4_video(pptx_path):
prs = Presentation(pptx_path)
for slide in prs.slides:
for shape in slide.shapes:
if shape.shape_type == 16:
if shape.media_type == 3:
return 1
return 0
if __name__ == "__main__":
path1 = "../../任务数据/LibreOffice Impress/Change_Color_Slide_Number_gold_textbox.pptx"
presentation = Presentation(path1)

44
desktop_env/evaluators/metrics/table.py
View File

@@ -3,6 +3,8 @@ import operator
from numbers import Number
from typing import Any, Union
from typing import Dict, List
import os.path
import itertools
import openpyxl
import pandas as pd
@@ -26,6 +28,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
* chart
* number_format
"chart_props": list of str, giving the converned chart properties
"as_shown": bool, TODO
}
Return:
@@ -35,10 +38,35 @@ def compare_table(actual: str, expected: str, **options) -> float:
if actual is None:
return 0.
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
logger.debug("Normal Contents Metric: {:}".format(metric))
if options.get("as_shown", False):
expected_csv: str = os.path.splitext(expected)[0] + ".csv"
actual_csv: str = os.path.splitext(actual)[0] + ".csv"
with open(expected_csv) as f:
expected_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0
, map( lambda l: l.strip()
, reversed(f.read().splitlines())
)
)
)
if options.get("ignore_case", False):
expected_lines = [l.lower() for l in expected_lines]
with open(actual_csv) as f:
actual_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0
, map( lambda l: l.strip()
, reversed(f.read().splitlines())
)
)
)
if options.get("ignore_case", False):
actual_lines = [l.lower() for l in actual_lines]
metric: bool = expected_lines==actual_lines
logger.debug("Content Metric just as shown: %s", metric)
else:
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
logger.debug("Normal Content Metric: {:}".format(metric))
features: List[str] = options.get("features", [])
for ftr in features:
@@ -219,6 +247,8 @@ if __name__ == '__main__':
# print(check_zoom(path1, {"relation": "lt", "ref_value": 100}))
# print(check_zoom(path2, {"relation": "lt", "ref_value": 100}))
path1 = "../../任务数据/LibreOffice Calc/Padding_Decimals_In_Formular_gold.xlsx"
data_frame: pd.DataFrame = pd.read_excel(path1)
print(data_frame)
path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx"
path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx"
#data_frame: pd.DataFrame = pd.read_excel(path1)
#print(data_frame)
print(compare_table(path1, path2, as_shown=True))

3
desktop_env/server/README.md
View File

@@ -1,6 +1,7 @@
<!-- vimc: call SyntaxRange#Include('```xml', '```', 'xml', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```css', '```', 'css', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```sh', '```', 'sh', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```bash', '```', 'sh', 'NonText'): -->
### About the Converted Accessibility Tree
@@ -57,7 +58,7 @@ contents.
An example of a CSS selector:
```css
application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]
application[name=Thunderbird] page-tab-list[attr|id="tabmail-tabs"]>page-tab[name="About Profiles"]
```
This selector will select the page tab of profile manager in Thunderbird (if open).

113
desktop_env/server/main.py
View File

@@ -14,7 +14,7 @@ import pyautogui
import requests
from PIL import Image
from Xlib import display, X
from flask import Flask, request, jsonify, send_file, abort, send_from_directory
from flask import Flask, request, jsonify, send_file, abort #, send_from_directory
from lxml.etree import _Element
from pyatspi import Accessible, StateType
from pyatspi import Action as ATAction
@@ -579,39 +579,114 @@ def open_file():
def activate_window():
data = request.json
window_name = data.get('window_name', None)
if not window_name:
return "window_name required", 400
strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string
by_class_name: bool = data.get("by_class", False)
os_name = platform.system()
if os_name == 'Windows':
import pygetwindow as gw
try:
# Find the VS Code window
vscode_window = gw.getWindowsWithTitle(window_name)[0]
# Activate the window, bringing it to the front
vscode_window.activate()
except IndexError:
return "VS Code window not found.", 404
if by_class_name:
return "Get window by class name is not supported on Windows currently.", 500
windows: List[gw.Window] = gw.getWindowsWithTitle(window_name)
window: Optional[gw.Window] = None
if len(windows)==0:
return "Window {:} not found (empty results)".format(window_name), 404
elif strict:
for wnd in windows:
if wnd.title==wnd:
window = wnd
if window is None:
return "Window {:} not found (strict mode).".format(window_name), 404
else:
window = windows[0]
window.activate()
elif os_name == 'Darwin':
import pygetwindow as gw
try:
# Find the VS Code window
vscode_window = gw.getWindowsWithTitle(window_name)[0]
# Un-minimize the window and then bring it to the front
vscode_window.unminimize()
vscode_window.activate()
except IndexError:
return "VS Code window not found.", 404
if by_class_name:
return "Get window by class name is not supported on macOS currently.", 500
# Find the VS Code window
windows = gw.getWindowsWithTitle(window_name)
window: Optional[gw.Window] = None
if len(windows)==0:
return "Window {:} not found (empty results)".format(window_name), 404
elif strict:
for wnd in windows:
if wnd.title==wnd:
window = wnd
if window is None:
return "Window {:} not found (strict mode).".format(window_name), 404
else:
window = windows[0]
# Un-minimize the window and then bring it to the front
window.unminimize()
window.activate()
elif os_name == 'Linux':
# Attempt to activate VS Code window using wmctrl
subprocess.Popen(["wmctrl", "-a", window_name])
subprocess.run( [ "wmctrl"
, "-{:}{:}a".format( "x" if by_class_name else ""
, "F" if strict else ""
)
, window_name
]
)
else:
return f"Operating system {os_name} not supported.", 400
return "File opened successfully", 200
return "Window activated successfully", 200
@app.route("/setup/close_window", methods=["POST"])
def close_window():
data = request.json
if "window_name" not in data:
return "window_name required", 400
window_name: str = data["window_name"]
strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string
by_class_name: bool = data.get("by_class", False)
os_name: str = platform.system()
if os_name == "Windows":
import pygetwindow as gw
if by_class_name:
return "Get window by class name is not supported on Windows currently.", 500
windows: List[gw.Window] = gw.getWindowsWithTitle(window_name)
window: Optional[gw.Window] = None
if len(windows)==0:
return "Window {:} not found (empty results)".format(window_name), 404
elif strict:
for wnd in windows:
if wnd.title==wnd:
window = wnd
if window is None:
return "Window {:} not found (strict mode).".format(window_name), 404
else:
window = windows[0]
window.close()
elif os_name == "Linux":
subprocess.run( [ "wmctrl"
, "-{:}{:}c".format( "x" if by_class_name else ""
, "F" if strict else ""
)
, window_name
]
)
elif os_name=="Darwin":
import pygetwindow as gw
return "Currently not supported on macOS.", 500
else:
return "Not supported platform {:}".format(os_name), 500
return "Window closed successfully.", 200
@app.route('/start_recording', methods=['POST'])
def start_recording():
@@ -639,7 +714,7 @@ def end_recording():
recording_process.terminate()
recording_process.wait()
return_code = recording_process.returncode
#return_code = recording_process.returncode
output, error = recording_process.communicate()
recording_process = None

103
evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json
View File

@@ -3,20 +3,101 @@
"snapshot": "libreoffice_calc",
"instruction": "I would like to pad all the numbers in the 'Old ID' column with zeros in front, to fill them up to seven digits in the 'New 7 Digit ID' column.",
"source": "https://www.youtube.com/shorts/FPAQaDTS8VY",
"config": {
"download": [
[
"",
"C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx"
]
],
"open": [
"C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx"
]
"config": [
{
"type": "download",
"parameters": {
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=1DqGy5JRKOuZMRJ8O76d4Cds4WaRyz8V1&export=download&authuser=0&confirm=t&uuid=fa0694d1-2a77-4fd2-89d3-d9b854317823&at=APZUnTU9BxqG7E8tLZ104c0E8BEL:1705501029016",
"path": "/home/user/Customers_New_7digit_Id.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Customers_New_7digit_Id.xlsx"
}
}
},
"trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a",
"related_apps": [
"libreoffice calc"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Customers_New_7digit_Id.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Customers_New_7digit_Id.xlsx",
"/home/user/Customers_New_7digit_Id.csv"
],
"dest": [
"Customers_New_7digit_Id.xlsx",
"Customers_New_7digit_Id.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=1zXz5k5A403IR0GE6DRRXRgQZrSIoVFSz&export=download&authuser=0&confirm=t&uuid=ba70b841-969c-4d91-9288-0011aeecf251&at=APZUnTWx3LL9udCgJAh-VMFzzfod:1705501007861",
"https://drive.usercontent.google.com/download?id=1h1GnUpyj92K7FXiHJ1xVaUYW_UYMDLPM&export=download&authuser=0&confirm=t&uuid=98de75bd-ba32-4ceb-97a7-b8d303a8dc96&at=APZUnTUewdfFIsyC3UlSlMcmmRbo:1705500978790"
],
"dest": [
"Customers_New_7digit_Id_gold.xlsx",
"Customers_New_7digit_Id_gold.csv"
],
"multi": true
},
"options": {
"as_shown": true
}
}
}

25
evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
View File

@@ -27,6 +27,31 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "copy_sheet_insert.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "check_sheet_list",
"result": {
"type": "vm_file",

25
evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
View File

@@ -27,6 +27,31 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Zoom_Out_Oversized_Cells.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "check_xlsx_zoom",
"result": {
"type": "vm_file",

71
evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
View File

@@ -27,21 +27,78 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Represent_in_millions_billions.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Represent_in_millions_billions.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/Represent_in_millions_billions.xlsx",
"dest": "Represent_in_millions_billions.xlsx"
"path": [
"/home/user/Represent_in_millions_billions.xlsx",
"/home/user/Represent_in_millions_billions.csv"
],
"dest": [
"Represent_in_millions_billions.xlsx",
"Represent_in_millions_billions.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995",
"dest": "Represent_in_millions_billions_gold.xlsx"
"path": [
"https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995",
"https://drive.usercontent.google.com/download?id=1CjzmJ4vqwPI7DMTR_6a8ytO-W-4xavBS&export=download&authuser=0&confirm=t&uuid=32c3430a-6d3d-47cc-bb5c-e683963b19f7&at=APZUnTWSBLNtkrZTix_BRl9gyaek:1705497493679"
],
"dest": [
"Represent_in_millions_billions_gold.xlsx",
"Represent_in_millions_billions_gold.csv"
],
"multi": true
},
"options": {
"features": [
"number_format"
]
"as_shown": true,
"ignore_case": true
}
}
}

25
evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
View File

@@ -26,6 +26,31 @@
"libreoffice calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",

25
evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
View File

@@ -27,6 +27,31 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Create_column_charts_using_statistics.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",

25
evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
View File

@@ -27,6 +27,31 @@
"libreoffice_calc"
],
"evaluators": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Freeze_row_column.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "check_xlsx_freeze",
"result": {
"type": "vm_file",

25
evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
View File

@@ -27,6 +27,31 @@
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",

89
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
View File

@@ -1,7 +1,7 @@
{
"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
"snapshot": "libreoffice_calc",
"instruction": "Could you help me setting decimal separator as a dot (.)?",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
"config": [
{
@@ -14,6 +14,12 @@
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Set_Decimal_Separator_Dot.xlsx"
}
}
],
"trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25",
@@ -21,24 +27,77 @@
"libreoffice_calc"
],
"evaluator": {
"func": "check_libre_locale",
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Set_Decimal_Separator_Dot.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/.config/libreoffice/4/user/registrymodifications.xcu",
"dest": "registrymodifications.xcu"
"path": [
"/home/user/Set_Decimal_Separator_Dot.xlsx",
"/home/user/Set_Decimal_Separator_Dot.csv"
],
"dest": [
"Set_Decimal_Separator_Dot.xlsx",
"Set_Decimal_Separator_Dot.csv"
],
"multi": true
},
"expected": {
"type": "rule",
"rules": {
"locale_set": [
"ru-*",
"de-*",
"fr-*",
"pt-*",
"es-*",
"it-*"
]
}
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975",
"https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507"
],
"dest": [
"Set_Decimal_Separator_Dot_gold.xlsx",
"Set_Decimal_Separator_Dot_gold.csv"
],
"multi": true
},
"options": {
"as_shown": true
}
}
}

85
evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup Normal file
View File

@@ -0,0 +1,85 @@
{
"id": "a01fbce3-2793-461f-ab86-43680ccbae25",
"snapshot": "libreoffice_calc",
"instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?",
"source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc",
"config": [],
"trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25",
"related_apps": [
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true",
"--outdir",
"/home/user",
"/home/user/Set_Decimal_Separator_Dot.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Set_Decimal_Separator_Dot.xlsx",
"/home/user/Set_Decimal_Separator_Dot.csv"
],
"dest": [
"Set_Decimal_Separator_Dot.xlsx",
"Set_Decimal_Separator_Dot.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975",
"https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507"
],
"dest": [
"Set_Decimal_Separator_Dot_gold.xlsx",
"Set_Decimal_Separator_Dot_gold.csv"
],
"multi": true
},
"options": {
"as_shown": true
}
}
}

4
evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json
View File

@@ -30,8 +30,8 @@
"func": "check_pdf_pages",
"result": {
"type": "vm_file",
"path": "/home/user/Resize_Cells_Fit_Page.xlsx",
"dest": "Resize_Cells_Fit_Page.xlsx"
"path": "/home/user/Resize_Cells_Fit_Page.pdf",
"dest": "Resize_Cells_Fit_Page.pdf"
},
"expected": {
"type": "rule",

25
evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
View File

@@ -27,6 +27,31 @@
"libreoffice calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Quarterly_Product_Sales_by_Zone.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
}
],
"func": "compare_table",
"expected": {
"type": "cloud_file",

40
evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json
View File

@@ -1,12 +1,42 @@
{
"id": "3b27600c-3668-4abd-8f84-7bcdebbccbdb",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me change the background color to blue 2 and apply it to all my slides.",
"instruction": "Please make the background blue on all my slides.",
"source": "https://www.libreofficehelp.com/change-slide-background-impress/#All_Slides",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aHMJzk2G8B_EqDlTAZLEiJ4h-ZsgA9UE&export=download&authuser=0&confirm=t&uuid=196a082d-5f08-4b3e-a64f-c021351f9cd8&at=APZUnTUXH4gvLvElvm9TtFhUJlIn:1705481007789",
"path": "Desktop/lec17-gui-events.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/lec17-gui-events.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
}
"evaluator": {
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1LU-wnmIqMQgwkdAUFBLE1wNkH4gSl3IR&export=download&authuser=0&confirm=t&uuid=74520405-4028-4fbe-bab8-d56dc82ffb6c&at=APZUnTU0dz5ZE5CcQry8IeY5_s1J:1705481009686",
"dest": "lec17-gui-events_Gold.docx"
},
"result": {
"type": "vm_file",
"path": "Desktop/lec17-gui-events.pptx",
"dest": "lec17-gui-events.pptx"
}
}
}

35
evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json
View File

@@ -1,12 +1,37 @@
{
"id": "ce88f674-ab7a-43da-9201-468d38539e4a",
"snapshot": "libreoffice_impress",
"instruction": "Could you help me change my slides to portrait (from landscape)?",
"instruction": "Please set my slides upright instead of sideways.",
"source": "https://justclickhere.co.uk/resources/change-slides-in-impress-to-portrait/",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1LErTnC_w_YPQVo84QK5sifww9xZ-Cq0X&export=download&authuser=0&confirm=t&uuid=81ff0aaf-9c2e-4342-b7ce-36e65dd2218e&at=APZUnTUmQKCTp2HUP0dOqYqD10G3:1705479016156",
"path": "Desktop/AM_Last_Page_Template.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/AM_Last_Page_Template.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
}
"evaluator": {
"func": "check_slide_orientation_Portrait",
"result": {
"type": "vm_file",
"path": "Desktop/AM_Last_Page_Template.pptx",
"dest": "AM_Last_Page_Template.pptx"
}
}
}

46
evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json
View File

@@ -1,12 +1,48 @@
{
"id": "f0a334af-f91b-4c03-b578-aac9bec2b543",
"snapshot": "libreoffice_impress",
"instruction": "Help me insert the video at VIDEO_PATH in the current slide.",
"instruction": "Insert the video Movie_countdown_2.mov on the Desktop into my current slide, please.",
"source": "https://www.libreofficehelp.com/insert-video-impress-presentation/#Inserting_a_Video_in_Impress",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1vvRkrxOK_sFPX9PLFniFqrdNEZ2pQnPP&export=download&authuser=0&confirm=t&uuid=71964a12-2d0a-4c71-9375-2f9ec15de1ad&at=APZUnTX_B-T2GeZPS7ZmchMQ6E7m:1705481285721",
"path": "Desktop/Movie_activities_TURKEY.pptx"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1S4lACtBb40Ff0DEjB2bG2tzr2qWwQLGd&export=download&authuser=0&confirm=t&uuid=a28c123e-5371-4e17-82c2-ed7b1f05b728&at=APZUnTW_rlUPV6mM4RjS0R6dMSv4:1705469776913",
"path": "Desktop/Movie_countdown_2.mov"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/Movie_activities_TURKEY.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
}
"evaluator": {
"func": "contains_mp4_video",
"result": {
"type": "vm_file",
"path": "Desktop/Movie_activities_TURKEY.pptx",
"dest": "Movie_activities_TURKEY.pptx"
}
}
}

18
evaluation_examples/examples/template.json
View File

@@ -3,11 +3,25 @@
"snapshot": "libreoffice_calc",
"instruction": "",
"source": "",
"config": [],
"config": [
{
"type": "",
"parameters": {}
}
],
"trajectory": "trajectories/",
"related_apps": [
"app1",
"app2"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [],
"func": "func",
"result": {
"type": ""
},
"expected": {
"type": ""
}
}
}

80
evaluation_examples/examples/thunderbird/030eeff7-b492-4218-b312-701ec99ee0cc.json Normal file
View File

@@ -0,0 +1,80 @@
{
"id": "030eeff7-b492-4218-b312-701ec99ee0cc",
"snapshot": "thunderbird",
"instruction": "When I reply to an email, it quotes the original message but offsets it with an indentation and \">\" character. I would like to quote the original message with no indentation, and no special character. Could you help me remove the indentation and \">\" for me?",
"source": "https://superuser.com/questions/1781004/how-do-i-remove-the-indentation-and-character-in-quoted-text-of-a-reply-mess",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/030eeff7-b492-4218-b312-701ec99ee0cc",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"unexpect": {
"mail.identity.id1.auto_quote": {
"method": "eq",
"ref": false
}
}
}
}
}
}

14
evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json
View File

@@ -45,12 +45,16 @@
"evaluator": {
"postconfig": [
{
"type": "command",
"type": "close_window",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
"window_name": "Message Filters",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

19
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json
View File

@@ -45,16 +45,17 @@
"evaluator": {
"postconfig": [
{
"type": "execute",
"type": "close_window",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

19
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup
View File

@@ -23,16 +23,17 @@
"evaluator": {
"postconfig": [
{
"type": "execute",
"type": "close_window",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

15
evaluation_examples/examples/thunderbird/480bcfea-d68f-4aaa-a0a9-2589ef319381.json
View File

@@ -45,12 +45,17 @@
"evaluator": {
"postconfig": [
{
"type": "command",
"type": "close_window",
"parameters": {
"command": ["wmctrl", "-xFc", "Mail.thunderbird"],
"until": {
"returncode": 1
}
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

15
evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json
View File

@@ -45,12 +45,17 @@
"evaluator": {
"postconfig": [
{
"type": "command",
"type": "close_window",
"parameters": {
"command": ["wmctrl", "-xFc", "Mail.thunderbird"],
"until": {
"returncode": 1
}
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

15
evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup
View File

@@ -23,12 +23,17 @@
"evaluator": {
"postconfig": [
{
"type": "command",
"type": "close_window",
"parameters": {
"command": ["wmctrl", "-xFc", "Mail.thunderbird"],
"until": {
"returncode": 1
}
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

24
evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
View File

@@ -1,7 +1,7 @@
{
"id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"snapshot": "thunderbird",
"instruction": "Help me access my gmail account with address \"xx@gmail.com\" and password \"xxx\"",
"instruction": "Help me access my outlook account with address \"anonym-x2024@outlook.com\" and password 'gTCI\";=@y7|QJ0nDa_kN3Sb&>' (without ')",
"source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird",
"config": [
{
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
"url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058",
"path": "/home/user/thunderbird-profile-blank.tar.gz"
}
]
}
@@ -23,7 +23,7 @@
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"/home/user/thunderbird-profile-blank.tar.gz",
"-C",
"/home/user/"
]
@@ -38,7 +38,7 @@
}
}
],
"trajectory": "trajectories/",
"trajectory": "trajectories/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"related_apps": [
"thunderbird"
],
@@ -49,7 +49,7 @@
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/firefox_decrypt.py"
}
]
@@ -84,14 +84,14 @@
"rules": {
"expect": [
{
"url": "imap://imap.gmail.com",
"user": "xx@gmail.com",
"password": "xxx"
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>"
},
{
"url": "smtp://smtp.gmail.com",
"user": "xx@gmail.com",
"password": "xxx"
"url": "smtp://smtp.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>"
}
]
}

79
evaluation_examples/examples/thunderbird/94760984-3ff5-41ee-8347-cf1af709fea0.json Normal file
View File

@@ -0,0 +1,79 @@
{
"id": "94760984-3ff5-41ee-8347-cf1af709fea0",
"snapshot": "thunderbird",
"instruction": "Considering I work late into the ight and use Thunderbird frequently, I find that a full dark mode would be easier on my eyes during those hours. Can you help me enable a complete dark mode in Thunderbird?",
"source": "https://superuser.com/questions/1757333/how-can-i-view-thunderbird-in-full-dark-mode",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/94760984-3ff5-41ee-8347-cf1af709fea0",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"extensions.activeThemeID": {
"method": "re",
"ref": "dark"
}
}
}
}
}

101
evaluation_examples/examples/thunderbird/99146c54-4f37-4ab8-9327-5f3291665e1e.json Normal file
View File

@@ -0,0 +1,101 @@
{
"id": "99146c54-4f37-4ab8-9327-5f3291665e1e",
"snapshot": "thunderbird",
"instruction": "Due to certain security considerations and the nature of my work, I prefer not to configure an incoming email service in Thunderbird. However, I still need to send emails. Can you help me set up Thunderbird to send emails from anonym-x2024@outlook.com without configuring its incoming email service?",
"source": "https://superuser.com/questions/1764409/how-to-send-email-with-thunderbird-without-configuring-an-incoming-email-service",
"config": [
{
"type": download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058",
"path": "/home/user/thunderbird-profile-blank.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile-blank.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/99146c54-4f37-4ab8-9327-5f3291665e1e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/firefox_decrypt.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/user/firefox_decrypt.py",
"/home/user/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"type": "rule",
"rules": {
"unexpect": [
{
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com"
}
],
"expect": [
{
"url": "smtp://smtp.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>"
}
]
}
}
}
}

2
evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
View File

@@ -49,7 +49,7 @@
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/firefox_decrypt.py"
}
]

90
evaluation_examples/examples/thunderbird/c9e7eaf2-b1a1-4efc-a982-721972fa9f02.json Normal file
View File

@@ -0,0 +1,90 @@
{
"id": "c9e7eaf2-b1a1-4efc-a982-721972fa9f02",
"snapshot": "thunderbird",
"instruction": "Thunderbird's message filters seem to only fire on Inbox automatically. If you want to filter on subfolders, you'd have to start this filter manually. I am wondering if the filter can be applied automatically. Could you help me apply automatic message filters to subfolders",
"source": "https://superuser.com/questions/544480/how-to-apply-automatic-message-filters-to-subfolders-too?noredirect=1&lq=1",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/c9e7eaf2-b1a1-4efc-a982-721972fa9f02",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.server.default.applyIncomingFilters": {
"method": "eq",
"ref": true
},
"mail.imap.use_status_for_biff": {
"method": "eq",
"ref": false
}
},
"unexpect": {
"mail.server.default.autosync_offline_stores": {
"method": "eq",
"ref": false
}
}
}
}
}
}

14
evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json
View File

@@ -45,12 +45,16 @@
"evaluator": {
"postconfig": [
{
"type": "command",
"type": "close_window",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
"window_name": "Message Filters",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],

14
evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup
View File

@@ -23,12 +23,16 @@
"evaluator": {
"postconfig": [
{
"type": "command",
"type": "close_window",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
"window_name": "Message Filters",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],