diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index 13d4ee8..9c6b559 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -298,11 +298,11 @@ class SetupController: # TODO raise NotImplementedError() - def _activate_window_setup(self, window_name: str): + def _activate_window_setup(self, window_name: str, strict: bool = False, by_class: bool = False): if not window_name: raise Exception(f"Setup Open - Invalid path ({window_name}).") - payload = json.dumps({"window_name": window_name}) + payload = json.dumps({"window_name": window_name, "strict": strict, "by_class": by_class}) headers = { 'Content-Type': 'application/json' } @@ -317,6 +317,25 @@ class SetupController: except requests.exceptions.RequestException as e: logger.error("An error occurred while trying to send the request: %s", e) + def _close_window_setup(self, window_name: str, strict: bool = False, by_class: bool = False): + if not window_name: + raise Exception(f"Setup Open - Invalid path ({window_name}).") + + payload = json.dumps({"window_name": window_name, "strict": strict, "by_class": by_class}) + headers = { + 'Content-Type': 'application/json' + } + + # send request to server to open file + try: + response = requests.post(self.http_server + "/setup" + "/close_window", headers=headers, data=payload) + if response.status_code == 200: + logger.info("Command executed successfully: %s", response.text) + else: + logger.error(f"Failed to close window {window_name}. Status code: %s", response.text) + except requests.exceptions.RequestException as e: + logger.error("An error occurred while trying to send the request: %s", e) + # Chrome setup def _chrome_open_tabs_setup(self, urls_to_open: List[str]): host = self.vm_ip diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 6714b0e..95c0a18 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,50 +1,92 @@ import os -from typing import Dict -from typing import Optional +from typing import Dict, List, Set +from typing import Optional, Any, Union import requests -def get_cloud_file(env, config: Dict[str, str]) -> str: +def get_cloud_file(env, config: Dict[str, Any]) -> Union[str, List[str]]: """ Config: - path (str): the url to download from - dest (str): file name of the downloaded file + path (str|List[str]): the url to download from + dest (str|List[str])): file name of the downloaded file + multi (bool) : optional. if path and dest are lists providing + information of multiple files. defaults to False + gives (List[int]): optional. defaults to [0]. which files are directly + returned to the metric. if len==1, str is returned; else, list is + returned. """ - _path = os.path.join(env.cache_dir, config["dest"]) - if os.path.exists(_path): - return _path + if not config.get("multi", False): + paths: List[str] = [config["path"]] + dests: List[str] = [config["dest"]] + else: + paths: List[str] = config["path"] + dests: List[str] = config["dest"] + cache_paths: List[str] = [] - url = config["path"] - response = requests.get(url, stream=True) - response.raise_for_status() + gives: Set[int] = set(config.get("gives", [0])) - with open(_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) + for i, (p, d) in enumerate(zip(paths, dests)): + _path = os.path.join(env.cache_dir, d) + if i in gives: + cache_paths.append(_path) - return _path + if os.path.exists(_path): + #return _path + continue + + url = p + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + return cache_paths[0] if len(cache_paths)==1 else cache_paths -def get_vm_file(env, config: Dict[str, str]) -> Optional[str]: +def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Optional[str]]]: """ Config: path (str): absolute path on the VM to fetch dest (str): file name of the downloaded file + multi (bool) : optional. if path and dest are lists providing + information of multiple files. defaults to False + gives (List[int]): optional. defaults to [0]. which files are directly + returned to the metric. if len==1, str is returned; else, list is + returned. """ - _path = os.path.join(env.cache_dir, config["dest"]) + if not config.get("multi", False): + paths: List[str] = [config["path"]] + dests: List[str] = [config["dest"]] + else: + paths: List[str] = config["path"] + dests: List[str] = config["dest"] + cache_paths: List[str] = [] - file = env.controller.get_file(config["path"]) - if file is None: - return None - # raise FileNotFoundError("File not found on VM: {:}".format(config["path"])) - with open(_path, "wb") as f: - f.write(file) + gives: Set[int] = set(config.get("gives", [0])) - return _path + for i, (p, d) in enumerate(zip(paths, dests)): + _path = os.path.join(env.cache_dir, d) + + file = env.controller.get_file(p) + if file is None: + #return None + # raise FileNotFoundError("File not found on VM: {:}".format(config["path"])) + if i in gives: + cache_paths.append(None) + continue + + if i in gives: + cache_paths.append(_path) + with open(_path, "wb") as f: + f.write(file) + + return cache_paths[0] if len(cache_paths)==1 else cache_paths def get_cache_file(env, config: Dict[str, str]) -> str: diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 59ebab2..fc46434 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -15,3 +15,4 @@ from .general import check_csv, check_accessibility_tree, check_list, run_sqlite from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed from .impress import check_slide_numbers_color, compare_pptx_files, check_for_two_lines, check_for_audio, check_formula_shape, check_file_exists +from .impress import check_slide_orientation_Portrait, contains_mp4_video \ No newline at end of file diff --git a/desktop_env/evaluators/metrics/impress.py b/desktop_env/evaluators/metrics/impress.py index 719ae56..749bc0e 100644 --- a/desktop_env/evaluators/metrics/impress.py +++ b/desktop_env/evaluators/metrics/impress.py @@ -94,6 +94,25 @@ def check_formula_shape(prs): return 1 return 0 +def check_slide_orientation_Portrait(pptx_path): + presentation = Presentation(pptx_path) + + slide_height = presentation.slide_height + slide_width = presentation.slide_width + + if slide_width < slide_height: + return 1 + return 0 + +def contains_mp4_video(pptx_path): + prs = Presentation(pptx_path) + for slide in prs.slides: + for shape in slide.shapes: + if shape.shape_type == 16: + if shape.media_type == 3: + return 1 + return 0 + if __name__ == "__main__": path1 = "../../任务数据/LibreOffice Impress/Change_Color_Slide_Number_gold_textbox.pptx" presentation = Presentation(path1) diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index 03daef5..e09b9f9 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -3,6 +3,8 @@ import operator from numbers import Number from typing import Any, Union from typing import Dict, List +import os.path +import itertools import openpyxl import pandas as pd @@ -26,6 +28,7 @@ def compare_table(actual: str, expected: str, **options) -> float: * chart * number_format "chart_props": list of str, giving the converned chart properties + "as_shown": bool, TODO } Return: @@ -35,10 +38,35 @@ def compare_table(actual: str, expected: str, **options) -> float: if actual is None: return 0. - df1 = pd.read_excel(expected) - df2 = pd.read_excel(actual) - metric: bool = df1.equals(df2) - logger.debug("Normal Contents Metric: {:}".format(metric)) + if options.get("as_shown", False): + expected_csv: str = os.path.splitext(expected)[0] + ".csv" + actual_csv: str = os.path.splitext(actual)[0] + ".csv" + + with open(expected_csv) as f: + expected_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0 + , map( lambda l: l.strip() + , reversed(f.read().splitlines()) + ) + ) + ) + if options.get("ignore_case", False): + expected_lines = [l.lower() for l in expected_lines] + with open(actual_csv) as f: + actual_lines: List[str] = list( itertools.dropwhile( lambda l: len(l)==0 + , map( lambda l: l.strip() + , reversed(f.read().splitlines()) + ) + ) + ) + if options.get("ignore_case", False): + actual_lines = [l.lower() for l in actual_lines] + metric: bool = expected_lines==actual_lines + logger.debug("Content Metric just as shown: %s", metric) + else: + df1 = pd.read_excel(expected) + df2 = pd.read_excel(actual) + metric: bool = df1.equals(df2) + logger.debug("Normal Content Metric: {:}".format(metric)) features: List[str] = options.get("features", []) for ftr in features: @@ -219,6 +247,8 @@ if __name__ == '__main__': # print(check_zoom(path1, {"relation": "lt", "ref_value": 100})) # print(check_zoom(path2, {"relation": "lt", "ref_value": 100})) - path1 = "../../任务数据/LibreOffice Calc/Padding_Decimals_In_Formular_gold.xlsx" - data_frame: pd.DataFrame = pd.read_excel(path1) - print(data_frame) + path1 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id.xlsx" + path2 = "../../任务数据/LibreOffice Calc/Customers_New_7digit_Id_gold.xlsx" + #data_frame: pd.DataFrame = pd.read_excel(path1) + #print(data_frame) + print(compare_table(path1, path2, as_shown=True)) diff --git a/desktop_env/server/README.md b/desktop_env/server/README.md index 1eacf5f..05f2997 100644 --- a/desktop_env/server/README.md +++ b/desktop_env/server/README.md @@ -1,6 +1,7 @@ + ### About the Converted Accessibility Tree @@ -57,7 +58,7 @@ contents. An example of a CSS selector: ```css -application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"] +application[name=Thunderbird] page-tab-list[attr|id="tabmail-tabs"]>page-tab[name="About Profiles"] ``` This selector will select the page tab of profile manager in Thunderbird (if open). diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index f1f11b7..13a5469 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -14,7 +14,7 @@ import pyautogui import requests from PIL import Image from Xlib import display, X -from flask import Flask, request, jsonify, send_file, abort, send_from_directory +from flask import Flask, request, jsonify, send_file, abort #, send_from_directory from lxml.etree import _Element from pyatspi import Accessible, StateType from pyatspi import Action as ATAction @@ -579,39 +579,114 @@ def open_file(): def activate_window(): data = request.json window_name = data.get('window_name', None) + if not window_name: + return "window_name required", 400 + strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string + by_class_name: bool = data.get("by_class", False) os_name = platform.system() if os_name == 'Windows': import pygetwindow as gw - try: - # Find the VS Code window - vscode_window = gw.getWindowsWithTitle(window_name)[0] - # Activate the window, bringing it to the front - vscode_window.activate() - except IndexError: - return "VS Code window not found.", 404 + if by_class_name: + return "Get window by class name is not supported on Windows currently.", 500 + windows: List[gw.Window] = gw.getWindowsWithTitle(window_name) + + window: Optional[gw.Window] = None + if len(windows)==0: + return "Window {:} not found (empty results)".format(window_name), 404 + elif strict: + for wnd in windows: + if wnd.title==wnd: + window = wnd + if window is None: + return "Window {:} not found (strict mode).".format(window_name), 404 + else: + window = windows[0] + window.activate() elif os_name == 'Darwin': import pygetwindow as gw - try: - # Find the VS Code window - vscode_window = gw.getWindowsWithTitle(window_name)[0] - # Un-minimize the window and then bring it to the front - vscode_window.unminimize() - vscode_window.activate() - except IndexError: - return "VS Code window not found.", 404 + if by_class_name: + return "Get window by class name is not supported on macOS currently.", 500 + # Find the VS Code window + windows = gw.getWindowsWithTitle(window_name) + + window: Optional[gw.Window] = None + if len(windows)==0: + return "Window {:} not found (empty results)".format(window_name), 404 + elif strict: + for wnd in windows: + if wnd.title==wnd: + window = wnd + if window is None: + return "Window {:} not found (strict mode).".format(window_name), 404 + else: + window = windows[0] + + # Un-minimize the window and then bring it to the front + window.unminimize() + window.activate() elif os_name == 'Linux': # Attempt to activate VS Code window using wmctrl - subprocess.Popen(["wmctrl", "-a", window_name]) + subprocess.run( [ "wmctrl" + , "-{:}{:}a".format( "x" if by_class_name else "" + , "F" if strict else "" + ) + , window_name + ] + ) else: return f"Operating system {os_name} not supported.", 400 - return "File opened successfully", 200 + return "Window activated successfully", 200 +@app.route("/setup/close_window", methods=["POST"]) +def close_window(): + data = request.json + if "window_name" not in data: + return "window_name required", 400 + window_name: str = data["window_name"] + strict: bool = data.get("strict", False) # compare case-sensitively and match the whole string + by_class_name: bool = data.get("by_class", False) + + os_name: str = platform.system() + if os_name == "Windows": + import pygetwindow as gw + + if by_class_name: + return "Get window by class name is not supported on Windows currently.", 500 + windows: List[gw.Window] = gw.getWindowsWithTitle(window_name) + + window: Optional[gw.Window] = None + if len(windows)==0: + return "Window {:} not found (empty results)".format(window_name), 404 + elif strict: + for wnd in windows: + if wnd.title==wnd: + window = wnd + if window is None: + return "Window {:} not found (strict mode).".format(window_name), 404 + else: + window = windows[0] + window.close() + elif os_name == "Linux": + subprocess.run( [ "wmctrl" + , "-{:}{:}c".format( "x" if by_class_name else "" + , "F" if strict else "" + ) + , window_name + ] + ) + elif os_name=="Darwin": + import pygetwindow as gw + return "Currently not supported on macOS.", 500 + else: + return "Not supported platform {:}".format(os_name), 500 + + return "Window closed successfully.", 200 @app.route('/start_recording', methods=['POST']) def start_recording(): @@ -639,7 +714,7 @@ def end_recording(): recording_process.terminate() recording_process.wait() - return_code = recording_process.returncode + #return_code = recording_process.returncode output, error = recording_process.communicate() recording_process = None diff --git a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json index 6f92ac5..524b2d1 100644 --- a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json +++ b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json @@ -3,20 +3,101 @@ "snapshot": "libreoffice_calc", "instruction": "I would like to pad all the numbers in the 'Old ID' column with zeros in front, to fill them up to seven digits in the 'New 7 Digit ID' column.", "source": "https://www.youtube.com/shorts/FPAQaDTS8VY", - "config": { - "download": [ - [ - "", - "C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx" - ] - ], - "open": [ - "C:\\Users\\tianbaox\\Desktop\\Customers_New_7digit_Id.xlsx" - ] + "config": [ + { + "type": "download", + "parameters": { + "file": [ + { + "url": "https://drive.usercontent.google.com/download?id=1DqGy5JRKOuZMRJ8O76d4Cds4WaRyz8V1&export=download&authuser=0&confirm=t&uuid=fa0694d1-2a77-4fd2-89d3-d9b854317823&at=APZUnTU9BxqG7E8tLZ104c0E8BEL:1705501029016", + "path": "/home/user/Customers_New_7digit_Id.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Customers_New_7digit_Id.xlsx" + } + } }, "trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a", "related_apps": [ "libreoffice calc" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Customers_New_7digit_Id.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/Customers_New_7digit_Id.xlsx", + "/home/user/Customers_New_7digit_Id.csv" + ], + "dest": [ + "Customers_New_7digit_Id.xlsx", + "Customers_New_7digit_Id.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.usercontent.google.com/download?id=1zXz5k5A403IR0GE6DRRXRgQZrSIoVFSz&export=download&authuser=0&confirm=t&uuid=ba70b841-969c-4d91-9288-0011aeecf251&at=APZUnTWx3LL9udCgJAh-VMFzzfod:1705501007861", + "https://drive.usercontent.google.com/download?id=1h1GnUpyj92K7FXiHJ1xVaUYW_UYMDLPM&export=download&authuser=0&confirm=t&uuid=98de75bd-ba32-4ceb-97a7-b8d303a8dc96&at=APZUnTUewdfFIsyC3UlSlMcmmRbo:1705500978790" + ], + "dest": [ + "Customers_New_7digit_Id_gold.xlsx", + "Customers_New_7digit_Id_gold.csv" + ], + "multi": true + }, + "options": { + "as_shown": true + } + } } diff --git a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json index a37eeb3..89203e0 100644 --- a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json +++ b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json @@ -27,6 +27,31 @@ "libreoffice_calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "copy_sheet_insert.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "check_sheet_list", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json index 5ed7c1a..76a1018 100644 --- a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json +++ b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json @@ -27,6 +27,31 @@ "libreoffice_calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Zoom_Out_Oversized_Cells.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "check_xlsx_zoom", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json index cd545b0..516dc91 100644 --- a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json +++ b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json @@ -27,21 +27,78 @@ "libreoffice_calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Represent_in_millions_billions.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Represent_in_millions_billions.xlsx" + ] + } + } + ], "func": "compare_table", "result": { "type": "vm_file", - "path": "/home/user/Represent_in_millions_billions.xlsx", - "dest": "Represent_in_millions_billions.xlsx" + "path": [ + "/home/user/Represent_in_millions_billions.xlsx", + "/home/user/Represent_in_millions_billions.csv" + ], + "dest": [ + "Represent_in_millions_billions.xlsx", + "Represent_in_millions_billions.csv" + ], + "multi": true }, "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995", - "dest": "Represent_in_millions_billions_gold.xlsx" + "path": [ + "https://drive.usercontent.google.com/download?id=1Jy6lZexhU5t0eW1GwXJ_csnwIe0Xiy9-&export=download&authuser=0&confirm=t&uuid=601701e7-9eb8-4ce8-83d5-35916094a15d&at=APZUnTW4WE-plIC5MmWTuFu24qLL:1703857882995", + "https://drive.usercontent.google.com/download?id=1CjzmJ4vqwPI7DMTR_6a8ytO-W-4xavBS&export=download&authuser=0&confirm=t&uuid=32c3430a-6d3d-47cc-bb5c-e683963b19f7&at=APZUnTWSBLNtkrZTix_BRl9gyaek:1705497493679" + ], + "dest": [ + "Represent_in_millions_billions_gold.xlsx", + "Represent_in_millions_billions_gold.csv" + ], + "multi": true }, "options": { - "features": [ - "number_format" - ] + "as_shown": true, + "ignore_case": true } } } diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index cf37625..acde630 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -26,6 +26,31 @@ "libreoffice calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "compare_table", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json index e8ba626..71e6bf3 100644 --- a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json +++ b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json @@ -27,6 +27,31 @@ "libreoffice_calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Create_column_charts_using_statistics.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "compare_table", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json index 6a9d412..a4faba9 100644 --- a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json +++ b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json @@ -27,6 +27,31 @@ "libreoffice_calc" ], "evaluators": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Freeze_row_column.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "check_xlsx_freeze", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json index f45c50c..94ca26f 100644 --- a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json +++ b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json @@ -27,6 +27,31 @@ "libreoffice_calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "compare_table", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json index 6ed124f..50dcc83 100644 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json @@ -1,7 +1,7 @@ { "id": "a01fbce3-2793-461f-ab86-43680ccbae25", "snapshot": "libreoffice_calc", - "instruction": "Could you help me setting decimal separator as a dot (.)?", + "instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?", "source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc", "config": [ { @@ -14,6 +14,12 @@ } ] } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Set_Decimal_Separator_Dot.xlsx" + } } ], "trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25", @@ -21,24 +27,77 @@ "libreoffice_calc" ], "evaluator": { - "func": "check_libre_locale", + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Set_Decimal_Separator_Dot.xlsx" + ] + } + } + ], + "func": "compare_table", "result": { "type": "vm_file", - "path": "/home/user/.config/libreoffice/4/user/registrymodifications.xcu", - "dest": "registrymodifications.xcu" + "path": [ + "/home/user/Set_Decimal_Separator_Dot.xlsx", + "/home/user/Set_Decimal_Separator_Dot.csv" + ], + "dest": [ + "Set_Decimal_Separator_Dot.xlsx", + "Set_Decimal_Separator_Dot.csv" + ], + "multi": true }, "expected": { - "type": "rule", - "rules": { - "locale_set": [ - "ru-*", - "de-*", - "fr-*", - "pt-*", - "es-*", - "it-*" - ] - } + "type": "cloud_file", + "path": [ + "https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975", + "https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507" + ], + "dest": [ + "Set_Decimal_Separator_Dot_gold.xlsx", + "Set_Decimal_Separator_Dot_gold.csv" + ], + "multi": true + }, + "options": { + "as_shown": true } } } diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup new file mode 100644 index 0000000..9844796 --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.nosetup @@ -0,0 +1,85 @@ +{ + "id": "a01fbce3-2793-461f-ab86-43680ccbae25", + "snapshot": "libreoffice_calc", + "instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?", + "source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc", + "config": [], + "trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", + "--outdir", + "/home/user", + "/home/user/Set_Decimal_Separator_Dot.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/Set_Decimal_Separator_Dot.xlsx", + "/home/user/Set_Decimal_Separator_Dot.csv" + ], + "dest": [ + "Set_Decimal_Separator_Dot.xlsx", + "Set_Decimal_Separator_Dot.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975", + "https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507" + ], + "dest": [ + "Set_Decimal_Separator_Dot_gold.xlsx", + "Set_Decimal_Separator_Dot_gold.csv" + ], + "multi": true + }, + "options": { + "as_shown": true + } + } +} diff --git a/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json b/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json index 3704a7b..c8812c2 100644 --- a/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json +++ b/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json @@ -30,8 +30,8 @@ "func": "check_pdf_pages", "result": { "type": "vm_file", - "path": "/home/user/Resize_Cells_Fit_Page.xlsx", - "dest": "Resize_Cells_Fit_Page.xlsx" + "path": "/home/user/Resize_Cells_Fit_Page.pdf", + "dest": "Resize_Cells_Fit_Page.pdf" }, "expected": { "type": "rule", diff --git a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index 8d96854..02e2a48 100644 --- a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -27,6 +27,31 @@ "libreoffice calc" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Quarterly_Product_Sales_by_Zone.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + ] + } + } + ], "func": "compare_table", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json b/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json index 784913c..615cabc 100644 --- a/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json +++ b/evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json @@ -1,12 +1,42 @@ { "id": "3b27600c-3668-4abd-8f84-7bcdebbccbdb", "snapshot": "libreoffice_impress", - "instruction": "Could you help me change the background color to blue 2 and apply it to all my slides.", + "instruction": "Please make the background blue on all my slides.", "source": "https://www.libreofficehelp.com/change-slide-background-impress/#All_Slides", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1aHMJzk2G8B_EqDlTAZLEiJ4h-ZsgA9UE&export=download&authuser=0&confirm=t&uuid=196a082d-5f08-4b3e-a64f-c021351f9cd8&at=APZUnTUXH4gvLvElvm9TtFhUJlIn:1705481007789", + "path": "Desktop/lec17-gui-events.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/lec17-gui-events.pptx" + } + } + ], "trajectory": "trajectories/", "related_apps": [ - "" + "libreoffice_impress" ], - "evaluator": "evaluation_dir" -} + "evaluator": { + "func": "compare_pptx_files", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1LU-wnmIqMQgwkdAUFBLE1wNkH4gSl3IR&export=download&authuser=0&confirm=t&uuid=74520405-4028-4fbe-bab8-d56dc82ffb6c&at=APZUnTU0dz5ZE5CcQry8IeY5_s1J:1705481009686", + "dest": "lec17-gui-events_Gold.docx" + }, + "result": { + "type": "vm_file", + "path": "Desktop/lec17-gui-events.pptx", + "dest": "lec17-gui-events.pptx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json b/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json index 1149fbb..7f3b272 100644 --- a/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json +++ b/evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json @@ -1,12 +1,37 @@ { "id": "ce88f674-ab7a-43da-9201-468d38539e4a", "snapshot": "libreoffice_impress", - "instruction": "Could you help me change my slides to portrait (from landscape)?", + "instruction": "Please set my slides upright instead of sideways.", "source": "https://justclickhere.co.uk/resources/change-slides-in-impress-to-portrait/", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1LErTnC_w_YPQVo84QK5sifww9xZ-Cq0X&export=download&authuser=0&confirm=t&uuid=81ff0aaf-9c2e-4342-b7ce-36e65dd2218e&at=APZUnTUmQKCTp2HUP0dOqYqD10G3:1705479016156", + "path": "Desktop/AM_Last_Page_Template.pptx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/AM_Last_Page_Template.pptx" + } + } + ], "trajectory": "trajectories/", "related_apps": [ - "" + "libreoffice_impress" ], - "evaluator": "evaluation_dir" -} + "evaluator": { + "func": "check_slide_orientation_Portrait", + "result": { + "type": "vm_file", + "path": "Desktop/AM_Last_Page_Template.pptx", + "dest": "AM_Last_Page_Template.pptx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json b/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json index 86342e6..3411aa8 100644 --- a/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json +++ b/evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json @@ -1,12 +1,48 @@ { "id": "f0a334af-f91b-4c03-b578-aac9bec2b543", "snapshot": "libreoffice_impress", - "instruction": "Help me insert the video at VIDEO_PATH in the current slide.", + "instruction": "Insert the video Movie_countdown_2.mov on the Desktop into my current slide, please.", "source": "https://www.libreofficehelp.com/insert-video-impress-presentation/#Inserting_a_Video_in_Impress", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1vvRkrxOK_sFPX9PLFniFqrdNEZ2pQnPP&export=download&authuser=0&confirm=t&uuid=71964a12-2d0a-4c71-9375-2f9ec15de1ad&at=APZUnTX_B-T2GeZPS7ZmchMQ6E7m:1705481285721", + "path": "Desktop/Movie_activities_TURKEY.pptx" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1S4lACtBb40Ff0DEjB2bG2tzr2qWwQLGd&export=download&authuser=0&confirm=t&uuid=a28c123e-5371-4e17-82c2-ed7b1f05b728&at=APZUnTW_rlUPV6mM4RjS0R6dMSv4:1705469776913", + "path": "Desktop/Movie_countdown_2.mov" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "Desktop/Movie_activities_TURKEY.pptx" + } + } + ], "trajectory": "trajectories/", "related_apps": [ - "" + "libreoffice_impress" ], - "evaluator": "evaluation_dir" -} + "evaluator": { + "func": "contains_mp4_video", + "result": { + "type": "vm_file", + "path": "Desktop/Movie_activities_TURKEY.pptx", + "dest": "Movie_activities_TURKEY.pptx" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/template.json b/evaluation_examples/examples/template.json index 35b0c6b..2b3a508 100644 --- a/evaluation_examples/examples/template.json +++ b/evaluation_examples/examples/template.json @@ -3,11 +3,25 @@ "snapshot": "libreoffice_calc", "instruction": "", "source": "", - "config": [], + "config": [ + { + "type": "", + "parameters": {} + } + ], "trajectory": "trajectories/", "related_apps": [ "app1", "app2" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [], + "func": "func", + "result": { + "type": "" + }, + "expected": { + "type": "" + } + } } diff --git a/evaluation_examples/examples/thunderbird/030eeff7-b492-4218-b312-701ec99ee0cc.json b/evaluation_examples/examples/thunderbird/030eeff7-b492-4218-b312-701ec99ee0cc.json new file mode 100644 index 0000000..a419f69 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/030eeff7-b492-4218-b312-701ec99ee0cc.json @@ -0,0 +1,80 @@ +{ + "id": "030eeff7-b492-4218-b312-701ec99ee0cc", + "snapshot": "thunderbird", + "instruction": "When I reply to an email, it quotes the original message but offsets it with an indentation and \">\" character. I would like to quote the original message with no indentation, and no special character. Could you help me remove the indentation and \">\" for me?", + "source": "https://superuser.com/questions/1781004/how-do-i-remove-the-indentation-and-character-in-quoted-text-of-a-reply-mess", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/030eeff7-b492-4218-b312-701ec99ee0cc", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "close_window", + "parameters": { + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "check_thunderbird_prefs", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js", + "dest": "thunder-prefs.js" + }, + "expected": { + "type": "rule", + "rules": { + "unexpect": { + "mail.identity.id1.auto_quote": { + "method": "eq", + "ref": false + } + } + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json b/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json index a1c3154..0f9c709 100644 --- a/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json +++ b/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json @@ -45,12 +45,16 @@ "evaluator": { "postconfig": [ { - "type": "command", + "type": "close_window", "parameters": { - "command": ["wmctrl", "-Fc", "Message Filters"], - "until": { - "returncode": 1 - } + "window_name": "Message Filters", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json index 4f688d6..f8d675b 100644 --- a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json @@ -45,16 +45,17 @@ "evaluator": { "postconfig": [ { - "type": "execute", + "type": "close_window", "parameters": { - "command": [ - "wmctrl", - "-xFc", - "Mail.thunderbird" - ], - "until": { - "returncode": 1 - } + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup index 7c1e728..46794f9 100644 --- a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup @@ -23,16 +23,17 @@ "evaluator": { "postconfig": [ { - "type": "execute", + "type": "close_window", "parameters": { - "command": [ - "wmctrl", - "-xFc", - "Mail.thunderbird" - ], - "until": { - "returncode": 1 - } + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/480bcfea-d68f-4aaa-a0a9-2589ef319381.json b/evaluation_examples/examples/thunderbird/480bcfea-d68f-4aaa-a0a9-2589ef319381.json index 7e95acb..8268fc8 100644 --- a/evaluation_examples/examples/thunderbird/480bcfea-d68f-4aaa-a0a9-2589ef319381.json +++ b/evaluation_examples/examples/thunderbird/480bcfea-d68f-4aaa-a0a9-2589ef319381.json @@ -45,12 +45,17 @@ "evaluator": { "postconfig": [ { - "type": "command", + "type": "close_window", "parameters": { - "command": ["wmctrl", "-xFc", "Mail.thunderbird"], - "until": { - "returncode": 1 - } + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json b/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json index 4e01ff5..369e8e7 100644 --- a/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json +++ b/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json @@ -45,12 +45,17 @@ "evaluator": { "postconfig": [ { - "type": "command", + "type": "close_window", "parameters": { - "command": ["wmctrl", "-xFc", "Mail.thunderbird"], - "until": { - "returncode": 1 - } + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup b/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup index f0a416a..4b6feef 100644 --- a/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup +++ b/evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup @@ -23,12 +23,17 @@ "evaluator": { "postconfig": [ { - "type": "command", + "type": "close_window", "parameters": { - "command": ["wmctrl", "-xFc", "Mail.thunderbird"], - "until": { - "returncode": 1 - } + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json index 142cd9f..5c7666a 100644 --- a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json +++ b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json @@ -1,7 +1,7 @@ { "id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", "snapshot": "thunderbird", - "instruction": "Help me access my gmail account with address \"xx@gmail.com\" and password \"xxx\"", + "instruction": "Help me access my outlook account with address \"anonym-x2024@outlook.com\" and password 'gTCI\";=@y7|QJ0nDa_kN3Sb&>' (without ')", "source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird", "config": [ { @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", - "path": "/home/user/thunderbird-profile.tar.gz" + "url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058", + "path": "/home/user/thunderbird-profile-blank.tar.gz" } ] } @@ -23,7 +23,7 @@ "-xzv", "--recursive-unlink", "-f", - "/home/user/thunderbird-profile.tar.gz", + "/home/user/thunderbird-profile-blank.tar.gz", "-C", "/home/user/" ] @@ -38,7 +38,7 @@ } } ], - "trajectory": "trajectories/", + "trajectory": "trajectories/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", "related_apps": [ "thunderbird" ], @@ -49,7 +49,7 @@ "parameters": { "files": [ { - "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", + "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py", "path": "/home/user/firefox_decrypt.py" } ] @@ -84,14 +84,14 @@ "rules": { "expect": [ { - "url": "imap://imap.gmail.com", - "user": "xx@gmail.com", - "password": "xxx" + "url": "imap://outlook.office365.com", + "user": "anonym-x2024@outlook.com", + "password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>" }, { - "url": "smtp://smtp.gmail.com", - "user": "xx@gmail.com", - "password": "xxx" + "url": "smtp://smtp.office365.com", + "user": "anonym-x2024@outlook.com", + "password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>" } ] } diff --git a/evaluation_examples/examples/thunderbird/94760984-3ff5-41ee-8347-cf1af709fea0.json b/evaluation_examples/examples/thunderbird/94760984-3ff5-41ee-8347-cf1af709fea0.json new file mode 100644 index 0000000..2d48f96 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/94760984-3ff5-41ee-8347-cf1af709fea0.json @@ -0,0 +1,79 @@ +{ + "id": "94760984-3ff5-41ee-8347-cf1af709fea0", + "snapshot": "thunderbird", + "instruction": "Considering I work late into the ight and use Thunderbird frequently, I find that a full dark mode would be easier on my eyes during those hours. Can you help me enable a complete dark mode in Thunderbird?", + "source": "https://superuser.com/questions/1757333/how-can-i-view-thunderbird-in-full-dark-mode", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/94760984-3ff5-41ee-8347-cf1af709fea0", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "close_window", + "parameters": { + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "check_thunderbird_prefs", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js", + "dest": "thunder-prefs.js" + }, + "expected": { + "type": "rule", + "rules": { + "expect": { + "extensions.activeThemeID": { + "method": "re", + "ref": "dark" + } + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/99146c54-4f37-4ab8-9327-5f3291665e1e.json b/evaluation_examples/examples/thunderbird/99146c54-4f37-4ab8-9327-5f3291665e1e.json new file mode 100644 index 0000000..923f7be --- /dev/null +++ b/evaluation_examples/examples/thunderbird/99146c54-4f37-4ab8-9327-5f3291665e1e.json @@ -0,0 +1,101 @@ +{ + "id": "99146c54-4f37-4ab8-9327-5f3291665e1e", + "snapshot": "thunderbird", + "instruction": "Due to certain security considerations and the nature of my work, I prefer not to configure an incoming email service in Thunderbird. However, I still need to send emails. Can you help me set up Thunderbird to send emails from anonym-x2024@outlook.com without configuring its incoming email service?", + "source": "https://superuser.com/questions/1764409/how-to-send-email-with-thunderbird-without-configuring-an-incoming-email-service", + "config": [ + { + "type": download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058", + "path": "/home/user/thunderbird-profile-blank.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile-blank.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/99146c54-4f37-4ab8-9327-5f3291665e1e", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py", + "path": "/home/user/firefox_decrypt.py" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python3", + "/home/user/firefox_decrypt.py", + "/home/user/.thunderbird", + "-n", + "-c", + "2", + "-f", + "csv", + "-d", + "," + ], + "stdout": "thunderbird-accounts.csv" + } + } + ], + "func": "check_csv", + "result": { + "type": "cache_file", + "path": "thunderbird-accounts.csv" + }, + "expected": { + "type": "rule", + "rules": { + "unexpect": [ + { + "url": "imap://outlook.office365.com", + "user": "anonym-x2024@outlook.com" + } + ], + "expect": [ + { + "url": "smtp://smtp.office365.com", + "user": "anonym-x2024@outlook.com", + "password": "gTCI\";=@y7|QJ0nDa_kN3Sb&>" + } + ] + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json index a50953e..54a4320 100644 --- a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json +++ b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json @@ -49,7 +49,7 @@ "parameters": { "files": [ { - "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", + "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py", "path": "/home/user/firefox_decrypt.py" } ] diff --git a/evaluation_examples/examples/thunderbird/c9e7eaf2-b1a1-4efc-a982-721972fa9f02.json b/evaluation_examples/examples/thunderbird/c9e7eaf2-b1a1-4efc-a982-721972fa9f02.json new file mode 100644 index 0000000..7e805d9 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/c9e7eaf2-b1a1-4efc-a982-721972fa9f02.json @@ -0,0 +1,90 @@ +{ + "id": "c9e7eaf2-b1a1-4efc-a982-721972fa9f02", + "snapshot": "thunderbird", + "instruction": "Thunderbird's message filters seem to only fire on Inbox automatically. If you want to filter on subfolders, you'd have to start this filter manually. I am wondering if the filter can be applied automatically. Could you help me apply automatic message filters to subfolders", + "source": "https://superuser.com/questions/544480/how-to-apply-automatic-message-filters-to-subfolders-too?noredirect=1&lq=1", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/c9e7eaf2-b1a1-4efc-a982-721972fa9f02", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "close_window", + "parameters": { + "window_name": "Mail.thunderbird", + "strict": true, + "by_class": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } + ], + "func": "check_thunderbird_prefs", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js", + "dest": "thunder-prefs.js" + }, + "expected": { + "type": "rule", + "rules": { + "expect": { + "mail.server.default.applyIncomingFilters": { + "method": "eq", + "ref": true + }, + "mail.imap.use_status_for_biff": { + "method": "eq", + "ref": false + } + }, + "unexpect": { + "mail.server.default.autosync_offline_stores": { + "method": "eq", + "ref": false + } + } + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json index 3dd986a..2a47a8c 100644 --- a/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json +++ b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json @@ -45,12 +45,16 @@ "evaluator": { "postconfig": [ { - "type": "command", + "type": "close_window", "parameters": { - "command": ["wmctrl", "-Fc", "Message Filters"], - "until": { - "returncode": 1 - } + "window_name": "Message Filters", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ], diff --git a/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup index 66af75d..b801b7a 100644 --- a/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup +++ b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup @@ -23,12 +23,16 @@ "evaluator": { "postconfig": [ { - "type": "command", + "type": "close_window", "parameters": { - "command": ["wmctrl", "-Fc", "Message Filters"], - "until": { - "returncode": 1 - } + "window_name": "Message Filters", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 } } ],