diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index 7dd884f..e0d9923 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -1,7 +1,7 @@ import json import logging import random -from typing import Any, Dict +from typing import Any, Dict, Optional import requests @@ -26,6 +26,15 @@ class PythonController: logger.error("Failed to get screenshot. Status code: %d", response.status_code) return None + def get_accessibility_tree(self) -> Optional[str]: + + response: requests.Response = requests.get(self.http_server + "/accessibility") + if response.status_code == 200: + return response.json()["AT"] + else: + logger.error("Failed to get accessibility tree. Status code: %d", response.status_code) + return None + def get_file(self, file_path: str): """ Gets a file from the server. @@ -73,7 +82,7 @@ class PythonController: if action_type == "MOVE_TO": if parameters == {} or None: - self.execute_python_command(f"pyautogui.moveTo()") + self.execute_python_command("pyautogui.moveTo()") elif "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] @@ -83,7 +92,7 @@ class PythonController: elif action_type == "CLICK": if parameters == {} or None: - self.execute_python_command(f"pyautogui.click()") + self.execute_python_command("pyautogui.click()") elif "button" in parameters and "x" in parameters and "y" in parameters: button = parameters["button"] x = parameters["x"] @@ -114,7 +123,7 @@ class PythonController: elif action_type == "MOUSE_DOWN": if parameters == {} or None: - self.execute_python_command(f"pyautogui.mouseDown()") + self.execute_python_command("pyautogui.mouseDown()") elif "button" in parameters: button = parameters["button"] self.execute_python_command(f"pyautogui.mouseDown(button='{button}')") @@ -123,7 +132,7 @@ class PythonController: elif action_type == "MOUSE_UP": if parameters == {} or None: - self.execute_python_command(f"pyautogui.mouseUp()") + self.execute_python_command("pyautogui.mouseUp()") elif "button" in parameters: button = parameters["button"] self.execute_python_command(f"pyautogui.mouseUp(button='{button}')") @@ -132,7 +141,7 @@ class PythonController: elif action_type == "RIGHT_CLICK": if parameters == {} or None: - self.execute_python_command(f"pyautogui.rightClick()") + self.execute_python_command("pyautogui.rightClick()") elif "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] @@ -142,7 +151,7 @@ class PythonController: elif action_type == "DOUBLE_CLICK": if parameters == {} or None: - self.execute_python_command(f"pyautogui.doubleClick()") + self.execute_python_command("pyautogui.doubleClick()") elif "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] @@ -208,7 +217,7 @@ class PythonController: raise Exception(f"Unknown parameters: {parameters}") keys = parameters["keys"] if not isinstance(keys, list): - raise Exception(f"Keys must be a list of keys") + raise Exception("Keys must be a list of keys") for key in keys: if key.lower() not in KEYBOARD_KEYS: raise Exception(f"Key must be one of {KEYBOARD_KEYS}") diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index cf97120..6067c6b 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -246,6 +246,9 @@ class SetupController: logger.error("An error occurred while trying to send the request: %s", e) traceback.print_exc() + def _command_setup(self, command: List[str], stdout: str = "", stderr: str = ""): + self._execute_setup(command, stdout, stderr) + def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]): # TODO raise NotImplementedError() diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index 91b99ca..c4c5f22 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -223,7 +223,7 @@ class DesktopEnv(gym.Env): Evaluate whether the task is successfully completed. """ - self.setup_controller.setup(self.evaluator["postconfig"]) if "postconfig" in self.evaluator else None + self.setup_controller.setup(self.evaluator.get("postconfig", [])) try: result_state = self.result_getter(self, self.evaluator["result"]) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index d91de89..7cc2c93 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,4 +1,5 @@ from .file import get_cloud_file, get_vm_file, get_cache_file from .misc import get_rule from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper +from .misc import get_rule, get_accessibility_tree from .vlc import get_vlc_playing_info, get_vlc_config diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index d969333..606fead 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,5 +1,6 @@ import os from typing import Dict +from typing import Optional import requests @@ -27,7 +28,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str: return _path -def get_vm_file(env, config: Dict[str, str]) -> str: +def get_vm_file(env, config: Dict[str, str]) -> Optional[str]: """ Config: path (str): absolute path on the VM to fetch @@ -37,10 +38,9 @@ def get_vm_file(env, config: Dict[str, str]) -> str: _path = os.path.join(env.cache_dir, config["dest"]) file = env.controller.get_file(config["path"]) - if file is None: - raise FileNotFoundError("File not found on VM: {:}".format(config["path"])) - + return None + #raise FileNotFoundError("File not found on VM: {:}".format(config["path"])) with open(_path, "wb") as f: f.write(file) diff --git a/desktop_env/evaluators/getters/general.py b/desktop_env/evaluators/getters/general.py new file mode 100644 index 0000000..22a20c7 --- /dev/null +++ b/desktop_env/evaluators/getters/general.py @@ -0,0 +1,23 @@ +from typing import Dict + +import os +import requests + + +def get_string(env, config: Dict[str, str]) -> str: + """ + Config: + string (str) + """ + + return config["string"] + +def get_command_line(env, config: Dict[str, str]) -> str: + """ + Config: + string (str) + """ + + f = os.popen(config["command"]) + + return f.read() \ No newline at end of file diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index a27cfa2..f4c7bf2 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -1,5 +1,6 @@ import logging from typing import TypeVar +#from typing import Dict, List logger = logging.getLogger("desktopenv.getters.misc") @@ -11,3 +12,8 @@ def get_rule(env, config: R) -> R: Returns the rule as-is. """ return config["rules"] + +def get_accessibility_tree(env, *args) -> str: + accessibility_tree: str = env.controller.get_accessibility_tree() + logger.debug("AT@eval: %s", accessibility_tree) + return accessibility_tree diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 95aea8d..f4b3360 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -6,4 +6,5 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im from .pdf import check_pdf_pages from .libreoffice import check_libre_locale from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, compare_videos -from .general import check_csv +from .general import check_csv, check_accessibility_tree, check_list + diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 37a3bc0..427198c 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -1,5 +1,17 @@ import csv -from typing import Dict, List + +import lxml.etree +from lxml.etree import _Element +from lxml.cssselect import CSSSelector + +from typing import Dict, List, Pattern +from typing import Callable, Any +from numbers import Number + +import operator +from rapidfuzz import fuzz +import functools +import re def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float: return all(k in item and item[k]==val for k, val in pattern.items()) @@ -22,9 +34,92 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: unexpect_metric = True with open(result) as f: reader = csv.DictReader(f) - + for rcd in reader: for i, r in enumerate(rules.get("expect", [])): expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd) - unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", [])) + unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", [])) return float(all(expect_metrics) and unexpect_metric) + +def check_list(result: str, rules: Dict[str, List[str]]) -> float: + """ + Args: + result (str): path to list file + rules (Dict[str, List[str]]): dict like + { + "expect": list of str as regexes + "unexpect": list of str as regexes + } + + Returns: + float + """ + + expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])] + unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])] + + expect_metrics = [False] * len(expect_patterns) + unexpect_metric = True + with open(result) as f: + for l in f: + for i, r in enumerate(expect_patterns): + expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None) + unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns) + return float(all(expect_metrics) and unexpect_metric) + +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } +def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: + """ + Args: + result (str): XML of GNOME Accessibility Tree + rules (Dict[str, Any]): dict like + { + "selectors": list of str as CSS selectors, will be connected by ", " + to form a composite selector. Only one from `selectors` and + `xpath` is needed. If both are present, `xpath` takes the + priority. + "xpath": str as xpath. Only one from `selectors` and `xpath` is + needed. If both are present, `xpath` takes the priority. + "text": str as the expected text content of the selected element. + "exact": bool specifying whether exact match or fuzzy match should + be performed. defaults to True + } + + Returns: + float + """ + + at: _Element = lxml.etree.fromstring(result) + if "xpath" in rules: + elements: List[_Element] = at.xpath(rules["xpath"], namespaces=_accessibility_ns_map) + elif "selectors" in rules: + selector = CSSSelector(", ".join(rules["selectors"]), namespaces=_accessibility_ns_map) + elements: List[_Element] = selector(at) + else: + raise ValueError("At least one of xpath and selectors is required") + + if len(elements)==0: + return 0. + + if "text" in rules: + match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio + , rules["text"] + ) + match_score: Number = 0 + for elm in elements: + match_score = max(match_score, match_func(elm.text or None)) + else: + match_score = 1. + + return float(match_score) + +#def check_existence(result: str, *args) -> float: + #return 1. - (result is None) diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index 34ef0b0..25b55f3 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -31,6 +31,9 @@ def compare_table(actual: str, expected: str, **options) -> float: float: the score """ + if actual is None: + return 0. + df1 = pd.read_excel(expected) df2 = pd.read_excel(actual) metric: bool = df1.equals(df2) @@ -71,6 +74,9 @@ def compare_table(actual: str, expected: str, **options) -> float: return float(metric) def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: + if result is None: + return 0. + # workbook: Workbook = openpyxl.load_workbook(filename=result) workbook = pd.ExcelFile(result) worksheet_names: List[str] = workbook.sheet_names @@ -109,10 +115,16 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: return float(passes) def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float: + if result is None: + return 0. + worksheet: Worksheet = openpyxl.load_workbook(filename=result).active return float(worksheet.freeze_panes == rules["position"]) def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float: + if result is None: + return 0. + worksheet = openpyxl.load_workbook(filename=result).active zoom_scale: Number = worksheet.sheet_view.zoomScale or 100. return float( getattr(operator, rules["relation"])( zoom_scale diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py new file mode 100644 index 0000000..581b8d1 --- /dev/null +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -0,0 +1,53 @@ +#from playwright.sync_api import sync_playwright, Browser +#from marionette_driver.marionette import Marionette +#import marionette +#import pyatspi + +import lxml.etree +from lxml.cssselect import CSSSelector +from lxml.etree import _Element + +from typing import List + +if __name__ == "__main__": + #with sync_playwright() as plwr: + #while True: + ##try: + #thunderbird: Browser = plwr.firefox.connect("http://127.0.0.1:6000", timeout=60) + #break + ##except: + ##pass + #for ctx in thunderbird.contexts: + #for p in ctx.pages: + #print(p.url) + + #thunderbird = Marionette() + #print(thunderbird.start_session()) + #print(thunderbird.chrome_window_handles) + #print(thunderbird.window_handles) + #print(thunderbird.current_chrome_window_handle) + #thunderbird.set_context(Marionette.CONTEXT_CONTENT) + #print(thunderbird.current_window_handle) + #thunderbird.switch_to_window(thunderbird.chrome_window_handles[0]) + #thunderbird.switch_to_default_content() + #thunderbird.switch_to_frame() + #print(thunderbird.get_url()) + #print(thunderbird.get_window_type()) + #thunderbird.fullscreen() + #print(thunderbird.close()) + + #registry = pyatspi.Registry.get_default() + #registry + + #xml = "../../任务数据/Thunderbird/vertical-card-view.xml" + xml = "../../任务数据/Thunderbird/vertical-table-view.xml" + at: _Element = lxml.etree.parse(xml) + + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + print(len(elements)) + for elm in elements: + print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True)) diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py index 9c5bb3b..9efef3e 100644 --- a/desktop_env/evaluators/metrics/vscode.py +++ b/desktop_env/evaluators/metrics/vscode.py @@ -20,5 +20,13 @@ def compare_text_file(actual: str, expected: str, **options) -> float: return 1.0 return 0.0 +def compare_answer(actual: str, expected: str, **options) -> float: + + if actual == expected: + return 1.0 + + # TODO: can use text embedding to get non-zero return + return 0.0 + if __name__ == '__main__': print(compare_text_file("README.md", "README.md")) \ No newline at end of file diff --git a/desktop_env/server/README.md b/desktop_env/server/README.md new file mode 100644 index 0000000..571081a --- /dev/null +++ b/desktop_env/server/README.md @@ -0,0 +1,73 @@ + + + + +### About the Converted Accessibility Tree + +For several applications like Firefox or Thunderbird, you should first enable + +```sh +gsettings set org.gnome.desktop.interface toolkit-accessibility true +``` + +to see their accessibility tree. + +#### Example of AT + +An example of a node: + +```xml +
+ 歡迎使用新的 Outlook.com 帳戶 +
+``` + +An example of a tree: + +```xml + + + ... + + ... + +``` + +#### Useful attributes + +1. `name` - shows the name of application, title of window, or name of some + component +2. `attr:class` - somewhat the same role as `class` in HTML +3. `attr:id` - somewhat the same role as `id` in HTML +4. `cp:screencoord` - absolute coordinator on the screen +5. `cp:windowcoord` - relative coordinator in the window +6. `cp:size` - the size + +Also several states like `st:enabled` and `st:visible` can be indicated. A full +state list is available at +. + +#### How to use it in evaluation + +See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and +function `check_accessibility_tree` in `metrics/general.py`. You can use CSS +selector or XPath to reference a target nodes. You can also check its text +contents. + +An example of a CSS selector: + +```css +application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"] +``` + +This selector will select the page tab of profile manager in Thunderbird (if open). + +For usage of CSS selector: . For usage of XPath: . + +#### Manual check + +You can use accerciser to check the accessibility tree on GNOME VM. + +```sh +sudo apt install accerciser +``` diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 1b9e42c..a49b565 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -3,18 +3,29 @@ import os import platform import subprocess from pathlib import Path -from typing import List + +import lxml.etree +from lxml.etree import _Element +import pyatspi +from pyatspi import Accessible, StateType +from pyatspi import Component, Document +from pyatspi import Text as ATText +from pyatspi import Value as ATValue +from pyatspi import Action as ATAction + +from typing import List, Dict +from typing import Any import Xlib import pyautogui -import requests from PIL import Image from Xlib import display, X +from pyxcursor import Xcursor + +import requests from flask import Flask, request, jsonify, send_file, abort from werkzeug.utils import secure_filename -from pyxcursor import Xcursor - app = Flask(__name__) pyautogui.PAUSE = 0 @@ -102,6 +113,141 @@ def capture_screen_with_cursor(): return send_file(file_path, mimetype='image/png') +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } +def _create_node(node: Accessible) -> _Element: + attribute_dict: Dict[str, Any] = {"name": node.name} + + # States {{{ # + states: List[StateType] = node.getState().get_states() + for st in states: + state_name: str = StateType._enum_lookup[st] + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["st"] + , state_name.split("_", maxsplit=1)[1].lower() + ) + ] = "true" + # }}} States # + + # Attributes {{{ # + attributes: List[str] = node.getAttributes() + for attrbt in attributes: + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["attr"] + , attribute_name + ) + ] = attribute_value + # }}} Attributes # + + # Component {{{ # + try: + component: Component = node.queryComponent() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN)) + attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW)) + attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT)) + attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize()) + # }}} Component # + + # Document {{{ # + try: + document: Document = node.queryDocument() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale() + attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount()) + attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber()) + for attrbt in document.getAttributes(): + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["docattr"] + , attribute_name + ) + ] = attribute_value + # }}} Document # + + # Text {{{ # + try: + text_obj: ATText = node.queryText() + except NotImplementedError: + pass + else: + # only text shown on current screen is available + #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount) + text: str = text_obj.getText(0, text_obj.characterCount) + # }}} Text # + + # Selection {{{ # + try: + node.querySelection() + except NotImplementedError: + pass + else: + attribute_dict["selection"] = "true" + # }}} Selection # + + # Value {{{ # + try: + value: ATValue = node.queryValue() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue) + attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue) + attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue) + attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement) + # }}} Value # + + # Action {{{ # + try: + action: ATAction = node.queryAction() + except NotImplementedError: + pass + else: + for i in range(action.nActions): + action_name: str = action.getName(i).replace(" ", "-") + attribute_dict[ "{{{:}}}{:}_desc"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getDescription(i) + attribute_dict[ "{{{:}}}{:}_kb"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getKeyBinding(i) + # }}} Action # + + xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-") + , attrib=attribute_dict + , nsmap=_accessibility_ns_map + ) + if "text" in locals() and len(text)>0: + xml_node.text = text + for ch in node: + xml_node.append(_create_node(ch)) + return xml_node + +@app.route("/accessibility", methods=["GET"]) +def get_accessibility_tree(): + desktop: Accessible = pyatspi.Registry.getDesktop(0) + desktop_xml: _Element = _create_node(desktop) + return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")}) @app.route('/screen_size', methods=['POST']) def get_screen_size(): diff --git a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json index bcde623..a37eeb3 100644 --- a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json +++ b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1ejNXBNOZtn64ugmvXot21pOEjx5xa-I5&export=download&authuser=0&confirm=t&uuid=61aa93e2-03f7-4b28-8e4a-cdff16a642f7&at=APZUnTVgPAHHfXaEjfKau5CDY1_K:1703509323791", - "path": "Desktop/copy_sheet_insert.xlsx" + "path": "/home/user/copy_sheet_insert.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/copy_sheet_insert.xlsx" + "path": "/home/user/copy_sheet_insert.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "check_sheet_list", "result": { "type": "vm_file", - "path": "Desktop/copy_sheet_insert.xlsx", + "path": "/home/user/copy_sheet_insert.xlsx", "dest": "copy_sheet_insert.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json index e38c502..5ed7c1a 100644 --- a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json +++ b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json @@ -10,7 +10,7 @@ "file": [ { "url": "https://drive.usercontent.google.com/download?id=1Wkepf_vic9o7CZFiosZ4jZT_Hy2WbRPZ&export=download&authuser=0&confirm=t&uuid=bc2ce901-a6bb-433f-bcce-cbe42d813f18&at=APZUnTVQcGTcXjwqenmtSH6IMFkM:1703858853235", - "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx" + "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx" + "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "check_xlsx_zoom", "result": { "type": "vm_file", - "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx", + "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx", "dest": "Zoom_Out_Oversized_Cells.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json index a54a774..cd545b0 100644 --- a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json +++ b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json @@ -10,7 +10,7 @@ "file": [ { "url": "https://drive.usercontent.google.com/download?id=16PowrQA4E71xUoJmpXPHy0dr9HBcTRmo&export=download&authuser=0&confirm=t&uuid=9a6265f7-585c-4cf8-b321-3b859aec1e68&at=APZUnTWzzOw85wws0ojXNPsIwnjE:1703858126178", - "path": "Desktop/Represent_in_millions_billions.xlsx" + "path": "/home/user/Represent_in_millions_billions.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Represent_in_millions_billions.xlsx" + "path": "/home/user/Represent_in_millions_billions.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "compare_table", "result": { "type": "vm_file", - "path": "Desktop/Represent_in_millions_billions.xlsx", + "path": "/home/user/Represent_in_millions_billions.xlsx", "dest": "Represent_in_millions_billions.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index fbf5a15..cf37625 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351", - "path": "Desktop/OrderId_Month_Chart.xlsx" + "path": "/home/user/OrderId_Month_Chart.xlsx" } ] } @@ -18,10 +18,9 @@ { "type": "open", "parameters": { - "path": "Desktop/OrderId_Month_Chart.xlsx" - } - } - ], + "path": "/home/user/OrderId_Month_Chart.xlsx" + ] + }, "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ "libreoffice calc" @@ -35,13 +34,12 @@ }, "result": { "type": "vm_file", - "path": "Desktop/OrderId_Month_Chart.xlsx", + "path": "/home/user/OrderId_Month_Chart.xlsx", "dest": "OrderId_Month_Chart.xlsx" }, "options": { "features": [ "sparkline" ] - } } } diff --git a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json index d359ca2..e8ba626 100644 --- a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json +++ b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1GOEacGTLP4EfGS8YwO9aGmmPgud5EavT&export=download&authuser=0&confirm=t&uuid=3971675c-3a76-4f89-863f-7f8afa59c3c5&at=APZUnTWaQ4_l1IiXsAR8VbjKf4uZ:1703595929357", - "path": "Desktop/Create_column_charts_using_statistics.xlsx" + "path": "/home/user/Create_column_charts_using_statistics.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Create_column_charts_using_statistics.xlsx" + "path": "/home/user/Create_column_charts_using_statistics.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "compare_table", "result": { "type": "vm_file", - "path": "Desktop/Create_column_charts_using_statistics.xlsx", + "path": "/home/user/Create_column_charts_using_statistics.xlsx", "dest": "Create_column_charts_using_statistics.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json index a16483e..b7c65d6 100644 --- a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json +++ b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json @@ -9,8 +9,8 @@ "parameters": { "files": [ { - "url": "", - "path": "Desktop/Employee_Roles_and_Ranks.xlsx" + "url": "https://101.43.24.67/s/FBip5fXoR4KEJaa", + "path": "/home/user/Employee_Roles_and_Ranks.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Employee_Roles_and_Ranks.xlsx" + "path": "/home/user/Employee_Roles_and_Ranks.xlsx" } } ], @@ -30,12 +30,12 @@ "func": "compare_table", "expected": { "type": "cloud_file", - "path": "", + "path": "https://101.43.24.67/s/wr7B4GeotNNoeHD", "dest": "Employee_Roles_and_Ranks_gold.xlsx" }, "result": { "type": "vm_file", - "path": "Desktop/Employee_Roles_and_Ranks.xlsx", + "path": "/home/user/Employee_Roles_and_Ranks.xlsx", "dest": "Employee_Roles_and_Ranks.xlsx" } } diff --git a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json index 14301da..6a9d412 100644 --- a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json +++ b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1ZhGLDKOden_oxzuN2xN9-jNQSHtCX6GE&export=download&authuser=0&confirm=t&uuid=2c097276-a610-4a9f-b6e4-5b54296c1555&at=APZUnTWc7zKPY_ykygn0mO1SAs4s:1703580957447", - "path": "Desktop/Freeze_row_column.xlsx" + "path": "/home/user/Freeze_row_column.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Freeze_row_column.xlsx" + "path": "/home/user/Freeze_row_column.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "check_xlsx_freeze", "result": { "type": "vm_file", - "path": "Desktop/Freeze_row_column.xlsx", + "path": "/home/user/Freeze_row_column.xlsx", "dest": "Freeze_row_column.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json index 14251e2..f45c50c 100644 --- a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json +++ b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json @@ -10,7 +10,7 @@ "file": [ { "url": "https://drive.usercontent.google.com/download?id=1LHxVvEpLI7kp0Iiy8K74gwkoGiwcLeYP&export=download&authuser=0&confirm=t&uuid=690287ee-d413-46e7-9b01-c56c12e445ff&at=APZUnTVCSd_ajhMGWpEgLHiExfbf:1704199487820", - "path": "/home/david/Padding_Decimals_In_Formular.xlsx" + "path": "/home/user/Padding_Decimals_In_Formular.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "/home/david/Padding_Decimals_In_Formular.xlsx" + "path": "/home/user/Padding_Decimals_In_Formular.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "compare_table", "result": { "type": "vm_file", - "path": "/home/david/Padding_Decimals_In_Formular_gold.xlsx", + "path": "/home/user/Padding_Decimals_In_Formular_gold.xlsx", "dest": "Padding_Decimals_In_Formular.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json index d99f170..6ed124f 100644 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json @@ -10,7 +10,7 @@ "file": [ { "url": "https://drive.usercontent.google.com/download?id=1uT0axjo9lwkKu6hYVnsAL2FCrdH0DLUv&export=download&authuser=0&confirm=t&uuid=e7da6304-9c7a-4862-8a30-9f2284b843da&at=APZUnTVNHThpAZJmF6IuPckFvslw:1704187618838", - "path": "/home/david/Set_Decimal_Separator_Dot.xlsx" + "path": "/home/user/Set_Decimal_Separator_Dot.xlsx" } ] } @@ -24,7 +24,7 @@ "func": "check_libre_locale", "result": { "type": "vm_file", - "path": "/home/david/.config/libreoffice/4/user/registrymodifications.xcu", + "path": "/home/user/.config/libreoffice/4/user/registrymodifications.xcu", "dest": "registrymodifications.xcu" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json b/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json index ff3d191..3704a7b 100644 --- a/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json +++ b/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json @@ -10,7 +10,7 @@ "file": [ { "url": "https://drive.usercontent.google.com/download?id=1O4bw7jEsVdFGeGeSX8hDjsvIbozV38sd&export=download&authuser=0&confirm=t&uuid=b6ceade0-e9c3-47bf-8c40-fef77b5ea1f1&at=APZUnTUUYaEx0Y_lAESeK1DfQZw6:1704179724348", - "path": "/home/david/Resize_Cells_Fit_Page.xlsx" + "path": "/home/user/Resize_Cells_Fit_Page.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "/home/david/Resize_Cells_Fit_Page.xlsx" + "path": "/home/user/Resize_Cells_Fit_Page.xlsx" } } ], @@ -30,7 +30,7 @@ "func": "check_pdf_pages", "result": { "type": "vm_file", - "path": "/home/david/Resize_Cells_Fit_Page.xlsx", + "path": "/home/user/Resize_Cells_Fit_Page.xlsx", "dest": "Resize_Cells_Fit_Page.xlsx" }, "expected": { diff --git a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index a9cae16..8d96854 100644 --- a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851", - "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx" + "path": "/home/user/Quarterly_Product_Sales_by_Zone.xlsx" } ] } @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx" + "path": "/home/user/Quarterly_Product_Sales_by_Zone.xlsx" } } ], @@ -35,7 +35,7 @@ }, "result": { "type": "vm_file", - "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx", + "path": "/home/user/Quarterly_Product_Sales_by_Zone.xlsx", "dest": "Quarterly_Product_Sales_by_Zone.xlsx" } } diff --git a/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json b/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json index ea4b832..969b193 100644 --- a/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json +++ b/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json @@ -1,12 +1,70 @@ { "id": "06fe7178-4491-4589-810f-2e2bc9502122", "snapshot": "thunderbird", - "instruction": "Could you help me back up all the email files in my profile to PROFILE_DIR?", + "instruction": "Could you help me back up all the email files in my profile to ~/email.bak? Please save them in eml format.", "source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird", - "config": [], - "trajectory": "trajectories/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/06fe7178-4491-4589-810f-2e2bc9502122", "related_apps": [ "thunderbird" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "command", + "parameters": { + "command": ["ls", "-R", "/home/user/emails.bak"], + "stdout": "emails.bak.ls" + } + } + ], + "func": "check_list", + "result": { + "type": "cache_file", + "path": "emails.bak.ls" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + "歡迎使用新的 Outlook.com 帳戶.*\\.eml", + "A Test E-mail.*\\.eml" + ] + } + } + } } diff --git a/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json index ce68b66..be50b20 100644 --- a/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json +++ b/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json @@ -3,10 +3,57 @@ "snapshot": "thunderbird", "instruction": "Could you help me open up the Thunderbird profile manager utility?", "source": "https://www.quora.com/How-do-I-open-a-Thunderbird-profile-manager-utility", - "config": [], - "trajectory": "trajectories/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/12086550-11c0-466b-b367-1d9e75b3910e", "related_apps": [ "thunderbird" ], - "evaluator": "evaluation_dir" + "evaluator": { + "result": { + "type": "accessibility_tree" + }, + "expected": { + "type": "rule", + "rules": { + "selectors": [ + "application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]" + ] + } + }, + "func": "check_accessibility_tree" + } } diff --git a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json index 449fa26..142cd9f 100644 --- a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json +++ b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", - "path": "/home/david/thunderbird-profile.tar.gz" + "path": "/home/user/thunderbird-profile.tar.gz" } ] } @@ -23,9 +23,9 @@ "-xzv", "--recursive-unlink", "-f", - "/home/david/thunderbird-profile.tar.gz", + "/home/user/thunderbird-profile.tar.gz", "-C", - "/home/david/" + "/home/user/" ] } }, @@ -50,7 +50,7 @@ "files": [ { "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", - "path": "/home/david/firefox_decrypt.py" + "path": "/home/user/firefox_decrypt.py" } ] } @@ -60,8 +60,8 @@ "parameters": { "command": [ "python3", - "/home/david/firefox_decrypt.py", - "/home/david/.thunderbird", + "/home/user/firefox_decrypt.py", + "/home/user/.thunderbird", "-n", "-c", "2", diff --git a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json index 7e977de..a50953e 100644 --- a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json +++ b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", - "path": "/home/david/thunderbird-profile.tar.gz" + "path": "/home/user/thunderbird-profile.tar.gz" } ] } @@ -23,9 +23,9 @@ "-xzv", "--recursive-unlink", "-f", - "/home/david/thunderbird-profile.tar.gz", + "/home/user/thunderbird-profile.tar.gz", "-C", - "/home/david/" + "/home/user/" ] } }, @@ -50,7 +50,7 @@ "files": [ { "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", - "path": "/home/david/firefox_decrypt.py" + "path": "/home/user/firefox_decrypt.py" } ] } @@ -60,8 +60,8 @@ "parameters": { "command": [ "python3", - "/home/david/firefox_decrypt.py", - "/home/david/.thunderbird", + "/home/user/firefox_decrypt.py", + "/home/user/.thunderbird", "-n", "-c", "2", diff --git a/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json b/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json index ce982b8..bf63054 100644 --- a/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json +++ b/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json @@ -9,5 +9,14 @@ "vscode" ], "evaluator": { + "func": "compare_answer", + "expected": { + "type": "string", + "string": "ms-python.python\n" + }, + "result": { + "type": "command_line", + "command": "code --list-extensions | grep ms-python.python" + } } } diff --git a/main.py b/main.py index ce1c0bd..f062e85 100644 --- a/main.py +++ b/main.py @@ -44,9 +44,9 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f: + with open("evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json", "r") as f: example = json.load(f) - example["snapshot"] = "base_setup3" + example["snapshot"] = "Snapshot 11" env = DesktopEnv( path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx", diff --git a/requirements.txt b/requirements.txt index 50a95a2..558098b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,4 +23,8 @@ python-docx python-pptx pypdf PyGetWindow +rapidfuzz +pyacoustid +opencv-python +ImageHash scikit-image