diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py
index 7dd884f..e0d9923 100644
--- a/desktop_env/controllers/python.py
+++ b/desktop_env/controllers/python.py
@@ -1,7 +1,7 @@
import json
import logging
import random
-from typing import Any, Dict
+from typing import Any, Dict, Optional
import requests
@@ -26,6 +26,15 @@ class PythonController:
logger.error("Failed to get screenshot. Status code: %d", response.status_code)
return None
+ def get_accessibility_tree(self) -> Optional[str]:
+
+ response: requests.Response = requests.get(self.http_server + "/accessibility")
+ if response.status_code == 200:
+ return response.json()["AT"]
+ else:
+ logger.error("Failed to get accessibility tree. Status code: %d", response.status_code)
+ return None
+
def get_file(self, file_path: str):
"""
Gets a file from the server.
@@ -73,7 +82,7 @@ class PythonController:
if action_type == "MOVE_TO":
if parameters == {} or None:
- self.execute_python_command(f"pyautogui.moveTo()")
+ self.execute_python_command("pyautogui.moveTo()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -83,7 +92,7 @@ class PythonController:
elif action_type == "CLICK":
if parameters == {} or None:
- self.execute_python_command(f"pyautogui.click()")
+ self.execute_python_command("pyautogui.click()")
elif "button" in parameters and "x" in parameters and "y" in parameters:
button = parameters["button"]
x = parameters["x"]
@@ -114,7 +123,7 @@ class PythonController:
elif action_type == "MOUSE_DOWN":
if parameters == {} or None:
- self.execute_python_command(f"pyautogui.mouseDown()")
+ self.execute_python_command("pyautogui.mouseDown()")
elif "button" in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.mouseDown(button='{button}')")
@@ -123,7 +132,7 @@ class PythonController:
elif action_type == "MOUSE_UP":
if parameters == {} or None:
- self.execute_python_command(f"pyautogui.mouseUp()")
+ self.execute_python_command("pyautogui.mouseUp()")
elif "button" in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.mouseUp(button='{button}')")
@@ -132,7 +141,7 @@ class PythonController:
elif action_type == "RIGHT_CLICK":
if parameters == {} or None:
- self.execute_python_command(f"pyautogui.rightClick()")
+ self.execute_python_command("pyautogui.rightClick()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -142,7 +151,7 @@ class PythonController:
elif action_type == "DOUBLE_CLICK":
if parameters == {} or None:
- self.execute_python_command(f"pyautogui.doubleClick()")
+ self.execute_python_command("pyautogui.doubleClick()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -208,7 +217,7 @@ class PythonController:
raise Exception(f"Unknown parameters: {parameters}")
keys = parameters["keys"]
if not isinstance(keys, list):
- raise Exception(f"Keys must be a list of keys")
+ raise Exception("Keys must be a list of keys")
for key in keys:
if key.lower() not in KEYBOARD_KEYS:
raise Exception(f"Key must be one of {KEYBOARD_KEYS}")
diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py
index cf97120..6067c6b 100644
--- a/desktop_env/controllers/setup.py
+++ b/desktop_env/controllers/setup.py
@@ -246,6 +246,9 @@ class SetupController:
logger.error("An error occurred while trying to send the request: %s", e)
traceback.print_exc()
+ def _command_setup(self, command: List[str], stdout: str = "", stderr: str = ""):
+ self._execute_setup(command, stdout, stderr)
+
def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]):
# TODO
raise NotImplementedError()
diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py
index 91b99ca..c4c5f22 100644
--- a/desktop_env/envs/desktop_env.py
+++ b/desktop_env/envs/desktop_env.py
@@ -223,7 +223,7 @@ class DesktopEnv(gym.Env):
Evaluate whether the task is successfully completed.
"""
- self.setup_controller.setup(self.evaluator["postconfig"]) if "postconfig" in self.evaluator else None
+ self.setup_controller.setup(self.evaluator.get("postconfig", []))
try:
result_state = self.result_getter(self, self.evaluator["result"])
diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py
index d91de89..7cc2c93 100644
--- a/desktop_env/evaluators/getters/__init__.py
+++ b/desktop_env/evaluators/getters/__init__.py
@@ -1,4 +1,5 @@
from .file import get_cloud_file, get_vm_file, get_cache_file
from .misc import get_rule
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
+from .misc import get_rule, get_accessibility_tree
from .vlc import get_vlc_playing_info, get_vlc_config
diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py
index d969333..606fead 100644
--- a/desktop_env/evaluators/getters/file.py
+++ b/desktop_env/evaluators/getters/file.py
@@ -1,5 +1,6 @@
import os
from typing import Dict
+from typing import Optional
import requests
@@ -27,7 +28,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str:
return _path
-def get_vm_file(env, config: Dict[str, str]) -> str:
+def get_vm_file(env, config: Dict[str, str]) -> Optional[str]:
"""
Config:
path (str): absolute path on the VM to fetch
@@ -37,10 +38,9 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
_path = os.path.join(env.cache_dir, config["dest"])
file = env.controller.get_file(config["path"])
-
if file is None:
- raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
-
+ return None
+ #raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
with open(_path, "wb") as f:
f.write(file)
diff --git a/desktop_env/evaluators/getters/general.py b/desktop_env/evaluators/getters/general.py
new file mode 100644
index 0000000..22a20c7
--- /dev/null
+++ b/desktop_env/evaluators/getters/general.py
@@ -0,0 +1,23 @@
+from typing import Dict
+
+import os
+import requests
+
+
+def get_string(env, config: Dict[str, str]) -> str:
+ """
+ Config:
+ string (str)
+ """
+
+ return config["string"]
+
+def get_command_line(env, config: Dict[str, str]) -> str:
+ """
+ Config:
+ string (str)
+ """
+
+ f = os.popen(config["command"])
+
+ return f.read()
\ No newline at end of file
diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py
index a27cfa2..f4c7bf2 100644
--- a/desktop_env/evaluators/getters/misc.py
+++ b/desktop_env/evaluators/getters/misc.py
@@ -1,5 +1,6 @@
import logging
from typing import TypeVar
+#from typing import Dict, List
logger = logging.getLogger("desktopenv.getters.misc")
@@ -11,3 +12,8 @@ def get_rule(env, config: R) -> R:
Returns the rule as-is.
"""
return config["rules"]
+
+def get_accessibility_tree(env, *args) -> str:
+ accessibility_tree: str = env.controller.get_accessibility_tree()
+ logger.debug("AT@eval: %s", accessibility_tree)
+ return accessibility_tree
diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py
index 95aea8d..f4b3360 100644
--- a/desktop_env/evaluators/metrics/__init__.py
+++ b/desktop_env/evaluators/metrics/__init__.py
@@ -6,4 +6,5 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
from .pdf import check_pdf_pages
from .libreoffice import check_libre_locale
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, compare_videos
-from .general import check_csv
+from .general import check_csv, check_accessibility_tree, check_list
+
diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py
index 37a3bc0..427198c 100644
--- a/desktop_env/evaluators/metrics/general.py
+++ b/desktop_env/evaluators/metrics/general.py
@@ -1,5 +1,17 @@
import csv
-from typing import Dict, List
+
+import lxml.etree
+from lxml.etree import _Element
+from lxml.cssselect import CSSSelector
+
+from typing import Dict, List, Pattern
+from typing import Callable, Any
+from numbers import Number
+
+import operator
+from rapidfuzz import fuzz
+import functools
+import re
def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
return all(k in item and item[k]==val for k, val in pattern.items())
@@ -22,9 +34,92 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
unexpect_metric = True
with open(result) as f:
reader = csv.DictReader(f)
-
+
for rcd in reader:
for i, r in enumerate(rules.get("expect", [])):
expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd)
- unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", []))
+ unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", []))
return float(all(expect_metrics) and unexpect_metric)
+
+def check_list(result: str, rules: Dict[str, List[str]]) -> float:
+ """
+ Args:
+ result (str): path to list file
+ rules (Dict[str, List[str]]): dict like
+ {
+ "expect": list of str as regexes
+ "unexpect": list of str as regexes
+ }
+
+ Returns:
+ float
+ """
+
+ expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])]
+ unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])]
+
+ expect_metrics = [False] * len(expect_patterns)
+ unexpect_metric = True
+ with open(result) as f:
+ for l in f:
+ for i, r in enumerate(expect_patterns):
+ expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None)
+ unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns)
+ return float(all(expect_metrics) and unexpect_metric)
+
+_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
+ , "attr": "uri:deskat:attributes.at-spi.gnome.org"
+ , "cp": "uri:deskat:component.at-spi.gnome.org"
+ , "doc": "uri:deskat:document.at-spi.gnome.org"
+ , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
+ , "txt": "uri:deskat:text.at-spi.gnome.org"
+ , "val": "uri:deskat:value.at-spi.gnome.org"
+ , "act": "uri:deskat:action.at-spi.gnome.org"
+ }
+def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
+ """
+ Args:
+ result (str): XML of GNOME Accessibility Tree
+ rules (Dict[str, Any]): dict like
+ {
+ "selectors": list of str as CSS selectors, will be connected by ", "
+ to form a composite selector. Only one from `selectors` and
+ `xpath` is needed. If both are present, `xpath` takes the
+ priority.
+ "xpath": str as xpath. Only one from `selectors` and `xpath` is
+ needed. If both are present, `xpath` takes the priority.
+ "text": str as the expected text content of the selected element.
+ "exact": bool specifying whether exact match or fuzzy match should
+ be performed. defaults to True
+ }
+
+ Returns:
+ float
+ """
+
+ at: _Element = lxml.etree.fromstring(result)
+ if "xpath" in rules:
+ elements: List[_Element] = at.xpath(rules["xpath"], namespaces=_accessibility_ns_map)
+ elif "selectors" in rules:
+ selector = CSSSelector(", ".join(rules["selectors"]), namespaces=_accessibility_ns_map)
+ elements: List[_Element] = selector(at)
+ else:
+ raise ValueError("At least one of xpath and selectors is required")
+
+ if len(elements)==0:
+ return 0.
+
+ if "text" in rules:
+ match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio
+ , rules["text"]
+ )
+ match_score: Number = 0
+ for elm in elements:
+ match_score = max(match_score, match_func(elm.text or None))
+ else:
+ match_score = 1.
+
+ return float(match_score)
+
+#def check_existence(result: str, *args) -> float:
+ #return 1. - (result is None)
diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py
index 34ef0b0..25b55f3 100644
--- a/desktop_env/evaluators/metrics/table.py
+++ b/desktop_env/evaluators/metrics/table.py
@@ -31,6 +31,9 @@ def compare_table(actual: str, expected: str, **options) -> float:
float: the score
"""
+ if actual is None:
+ return 0.
+
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
@@ -71,6 +74,9 @@ def compare_table(actual: str, expected: str, **options) -> float:
return float(metric)
def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
+ if result is None:
+ return 0.
+
# workbook: Workbook = openpyxl.load_workbook(filename=result)
workbook = pd.ExcelFile(result)
worksheet_names: List[str] = workbook.sheet_names
@@ -109,10 +115,16 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
return float(passes)
def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
+ if result is None:
+ return 0.
+
worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
return float(worksheet.freeze_panes == rules["position"])
def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
+ if result is None:
+ return 0.
+
worksheet = openpyxl.load_workbook(filename=result).active
zoom_scale: Number = worksheet.sheet_view.zoomScale or 100.
return float( getattr(operator, rules["relation"])( zoom_scale
diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py
new file mode 100644
index 0000000..581b8d1
--- /dev/null
+++ b/desktop_env/evaluators/metrics/thunderbird.py
@@ -0,0 +1,53 @@
+#from playwright.sync_api import sync_playwright, Browser
+#from marionette_driver.marionette import Marionette
+#import marionette
+#import pyatspi
+
+import lxml.etree
+from lxml.cssselect import CSSSelector
+from lxml.etree import _Element
+
+from typing import List
+
+if __name__ == "__main__":
+ #with sync_playwright() as plwr:
+ #while True:
+ ##try:
+ #thunderbird: Browser = plwr.firefox.connect("http://127.0.0.1:6000", timeout=60)
+ #break
+ ##except:
+ ##pass
+ #for ctx in thunderbird.contexts:
+ #for p in ctx.pages:
+ #print(p.url)
+
+ #thunderbird = Marionette()
+ #print(thunderbird.start_session())
+ #print(thunderbird.chrome_window_handles)
+ #print(thunderbird.window_handles)
+ #print(thunderbird.current_chrome_window_handle)
+ #thunderbird.set_context(Marionette.CONTEXT_CONTENT)
+ #print(thunderbird.current_window_handle)
+ #thunderbird.switch_to_window(thunderbird.chrome_window_handles[0])
+ #thunderbird.switch_to_default_content()
+ #thunderbird.switch_to_frame()
+ #print(thunderbird.get_url())
+ #print(thunderbird.get_window_type())
+ #thunderbird.fullscreen()
+ #print(thunderbird.close())
+
+ #registry = pyatspi.Registry.get_default()
+ #registry
+
+ #xml = "../../任务数据/Thunderbird/vertical-card-view.xml"
+ xml = "../../任务数据/Thunderbird/vertical-table-view.xml"
+ at: _Element = lxml.etree.parse(xml)
+
+ #elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags
+ #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page
+ #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page
+ #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
+ elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
+ print(len(elements))
+ for elm in elements:
+ print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))
diff --git a/desktop_env/evaluators/metrics/vscode.py b/desktop_env/evaluators/metrics/vscode.py
index 9c5bb3b..9efef3e 100644
--- a/desktop_env/evaluators/metrics/vscode.py
+++ b/desktop_env/evaluators/metrics/vscode.py
@@ -20,5 +20,13 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
return 1.0
return 0.0
+def compare_answer(actual: str, expected: str, **options) -> float:
+
+ if actual == expected:
+ return 1.0
+
+ # TODO: can use text embedding to get non-zero return
+ return 0.0
+
if __name__ == '__main__':
print(compare_text_file("README.md", "README.md"))
\ No newline at end of file
diff --git a/desktop_env/server/README.md b/desktop_env/server/README.md
new file mode 100644
index 0000000..571081a
--- /dev/null
+++ b/desktop_env/server/README.md
@@ -0,0 +1,73 @@
+
+
+
+
+### About the Converted Accessibility Tree
+
+For several applications like Firefox or Thunderbird, you should first enable
+
+```sh
+gsettings set org.gnome.desktop.interface toolkit-accessibility true
+```
+
+to see their accessibility tree.
+
+#### Example of AT
+
+An example of a node:
+
+```xml
+
+ 歡迎使用新的 Outlook.com 帳戶
+
+```
+
+An example of a tree:
+
+```xml
+
+
+ ...
+
+ ...
+
+```
+
+#### Useful attributes
+
+1. `name` - shows the name of application, title of window, or name of some
+ component
+2. `attr:class` - somewhat the same role as `class` in HTML
+3. `attr:id` - somewhat the same role as `id` in HTML
+4. `cp:screencoord` - absolute coordinator on the screen
+5. `cp:windowcoord` - relative coordinator in the window
+6. `cp:size` - the size
+
+Also several states like `st:enabled` and `st:visible` can be indicated. A full
+state list is available at
+.
+
+#### How to use it in evaluation
+
+See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and
+function `check_accessibility_tree` in `metrics/general.py`. You can use CSS
+selector or XPath to reference a target nodes. You can also check its text
+contents.
+
+An example of a CSS selector:
+
+```css
+application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]
+```
+
+This selector will select the page tab of profile manager in Thunderbird (if open).
+
+For usage of CSS selector: . For usage of XPath: .
+
+#### Manual check
+
+You can use accerciser to check the accessibility tree on GNOME VM.
+
+```sh
+sudo apt install accerciser
+```
diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py
index 1b9e42c..a49b565 100644
--- a/desktop_env/server/main.py
+++ b/desktop_env/server/main.py
@@ -3,18 +3,29 @@ import os
import platform
import subprocess
from pathlib import Path
-from typing import List
+
+import lxml.etree
+from lxml.etree import _Element
+import pyatspi
+from pyatspi import Accessible, StateType
+from pyatspi import Component, Document
+from pyatspi import Text as ATText
+from pyatspi import Value as ATValue
+from pyatspi import Action as ATAction
+
+from typing import List, Dict
+from typing import Any
import Xlib
import pyautogui
-import requests
from PIL import Image
from Xlib import display, X
+from pyxcursor import Xcursor
+
+import requests
from flask import Flask, request, jsonify, send_file, abort
from werkzeug.utils import secure_filename
-from pyxcursor import Xcursor
-
app = Flask(__name__)
pyautogui.PAUSE = 0
@@ -102,6 +113,141 @@ def capture_screen_with_cursor():
return send_file(file_path, mimetype='image/png')
+_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
+ , "attr": "uri:deskat:attributes.at-spi.gnome.org"
+ , "cp": "uri:deskat:component.at-spi.gnome.org"
+ , "doc": "uri:deskat:document.at-spi.gnome.org"
+ , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
+ , "txt": "uri:deskat:text.at-spi.gnome.org"
+ , "val": "uri:deskat:value.at-spi.gnome.org"
+ , "act": "uri:deskat:action.at-spi.gnome.org"
+ }
+def _create_node(node: Accessible) -> _Element:
+ attribute_dict: Dict[str, Any] = {"name": node.name}
+
+ # States {{{ #
+ states: List[StateType] = node.getState().get_states()
+ for st in states:
+ state_name: str = StateType._enum_lookup[st]
+ attribute_dict[ "{{{:}}}{:}"\
+ .format( _accessibility_ns_map["st"]
+ , state_name.split("_", maxsplit=1)[1].lower()
+ )
+ ] = "true"
+ # }}} States #
+
+ # Attributes {{{ #
+ attributes: List[str] = node.getAttributes()
+ for attrbt in attributes:
+ attribute_name: str
+ attribute_value: str
+ attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
+ attribute_dict[ "{{{:}}}{:}"\
+ .format( _accessibility_ns_map["attr"]
+ , attribute_name
+ )
+ ] = attribute_value
+ # }}} Attributes #
+
+ # Component {{{ #
+ try:
+ component: Component = node.queryComponent()
+ except NotImplementedError:
+ pass
+ else:
+ attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN))
+ attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW))
+ attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT))
+ attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize())
+ # }}} Component #
+
+ # Document {{{ #
+ try:
+ document: Document = node.queryDocument()
+ except NotImplementedError:
+ pass
+ else:
+ attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale()
+ attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount())
+ attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber())
+ for attrbt in document.getAttributes():
+ attribute_name: str
+ attribute_value: str
+ attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
+ attribute_dict[ "{{{:}}}{:}"\
+ .format( _accessibility_ns_map["docattr"]
+ , attribute_name
+ )
+ ] = attribute_value
+ # }}} Document #
+
+ # Text {{{ #
+ try:
+ text_obj: ATText = node.queryText()
+ except NotImplementedError:
+ pass
+ else:
+ # only text shown on current screen is available
+ #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
+ text: str = text_obj.getText(0, text_obj.characterCount)
+ # }}} Text #
+
+ # Selection {{{ #
+ try:
+ node.querySelection()
+ except NotImplementedError:
+ pass
+ else:
+ attribute_dict["selection"] = "true"
+ # }}} Selection #
+
+ # Value {{{ #
+ try:
+ value: ATValue = node.queryValue()
+ except NotImplementedError:
+ pass
+ else:
+ attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
+ attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
+ attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
+ attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
+ # }}} Value #
+
+ # Action {{{ #
+ try:
+ action: ATAction = node.queryAction()
+ except NotImplementedError:
+ pass
+ else:
+ for i in range(action.nActions):
+ action_name: str = action.getName(i).replace(" ", "-")
+ attribute_dict[ "{{{:}}}{:}_desc"\
+ .format( _accessibility_ns_map["act"]
+ , action_name
+ )
+ ] = action.getDescription(i)
+ attribute_dict[ "{{{:}}}{:}_kb"\
+ .format( _accessibility_ns_map["act"]
+ , action_name
+ )
+ ] = action.getKeyBinding(i)
+ # }}} Action #
+
+ xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-")
+ , attrib=attribute_dict
+ , nsmap=_accessibility_ns_map
+ )
+ if "text" in locals() and len(text)>0:
+ xml_node.text = text
+ for ch in node:
+ xml_node.append(_create_node(ch))
+ return xml_node
+
+@app.route("/accessibility", methods=["GET"])
+def get_accessibility_tree():
+ desktop: Accessible = pyatspi.Registry.getDesktop(0)
+ desktop_xml: _Element = _create_node(desktop)
+ return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
@app.route('/screen_size', methods=['POST'])
def get_screen_size():
diff --git a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
index bcde623..a37eeb3 100644
--- a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
+++ b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ejNXBNOZtn64ugmvXot21pOEjx5xa-I5&export=download&authuser=0&confirm=t&uuid=61aa93e2-03f7-4b28-8e4a-cdff16a642f7&at=APZUnTVgPAHHfXaEjfKau5CDY1_K:1703509323791",
- "path": "Desktop/copy_sheet_insert.xlsx"
+ "path": "/home/user/copy_sheet_insert.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/copy_sheet_insert.xlsx"
+ "path": "/home/user/copy_sheet_insert.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "check_sheet_list",
"result": {
"type": "vm_file",
- "path": "Desktop/copy_sheet_insert.xlsx",
+ "path": "/home/user/copy_sheet_insert.xlsx",
"dest": "copy_sheet_insert.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
index e38c502..5ed7c1a 100644
--- a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
+++ b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json
@@ -10,7 +10,7 @@
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=1Wkepf_vic9o7CZFiosZ4jZT_Hy2WbRPZ&export=download&authuser=0&confirm=t&uuid=bc2ce901-a6bb-433f-bcce-cbe42d813f18&at=APZUnTVQcGTcXjwqenmtSH6IMFkM:1703858853235",
- "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx"
+ "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx"
+ "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "check_xlsx_zoom",
"result": {
"type": "vm_file",
- "path": "Desktop/Zoom_Out_Oversized_Cells.xlsx",
+ "path": "/home/user/Zoom_Out_Oversized_Cells.xlsx",
"dest": "Zoom_Out_Oversized_Cells.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
index a54a774..cd545b0 100644
--- a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
+++ b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json
@@ -10,7 +10,7 @@
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=16PowrQA4E71xUoJmpXPHy0dr9HBcTRmo&export=download&authuser=0&confirm=t&uuid=9a6265f7-585c-4cf8-b321-3b859aec1e68&at=APZUnTWzzOw85wws0ojXNPsIwnjE:1703858126178",
- "path": "Desktop/Represent_in_millions_billions.xlsx"
+ "path": "/home/user/Represent_in_millions_billions.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Represent_in_millions_billions.xlsx"
+ "path": "/home/user/Represent_in_millions_billions.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "compare_table",
"result": {
"type": "vm_file",
- "path": "Desktop/Represent_in_millions_billions.xlsx",
+ "path": "/home/user/Represent_in_millions_billions.xlsx",
"dest": "Represent_in_millions_billions.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
index fbf5a15..cf37625 100644
--- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
+++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351",
- "path": "Desktop/OrderId_Month_Chart.xlsx"
+ "path": "/home/user/OrderId_Month_Chart.xlsx"
}
]
}
@@ -18,10 +18,9 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/OrderId_Month_Chart.xlsx"
- }
- }
- ],
+ "path": "/home/user/OrderId_Month_Chart.xlsx"
+ ]
+ },
"trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
"related_apps": [
"libreoffice calc"
@@ -35,13 +34,12 @@
},
"result": {
"type": "vm_file",
- "path": "Desktop/OrderId_Month_Chart.xlsx",
+ "path": "/home/user/OrderId_Month_Chart.xlsx",
"dest": "OrderId_Month_Chart.xlsx"
},
"options": {
"features": [
"sparkline"
]
- }
}
}
diff --git a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
index d359ca2..e8ba626 100644
--- a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
+++ b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1GOEacGTLP4EfGS8YwO9aGmmPgud5EavT&export=download&authuser=0&confirm=t&uuid=3971675c-3a76-4f89-863f-7f8afa59c3c5&at=APZUnTWaQ4_l1IiXsAR8VbjKf4uZ:1703595929357",
- "path": "Desktop/Create_column_charts_using_statistics.xlsx"
+ "path": "/home/user/Create_column_charts_using_statistics.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Create_column_charts_using_statistics.xlsx"
+ "path": "/home/user/Create_column_charts_using_statistics.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "compare_table",
"result": {
"type": "vm_file",
- "path": "Desktop/Create_column_charts_using_statistics.xlsx",
+ "path": "/home/user/Create_column_charts_using_statistics.xlsx",
"dest": "Create_column_charts_using_statistics.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json
index a16483e..b7c65d6 100644
--- a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json
+++ b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json
@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
- "url": "",
- "path": "Desktop/Employee_Roles_and_Ranks.xlsx"
+ "url": "https://101.43.24.67/s/FBip5fXoR4KEJaa",
+ "path": "/home/user/Employee_Roles_and_Ranks.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Employee_Roles_and_Ranks.xlsx"
+ "path": "/home/user/Employee_Roles_and_Ranks.xlsx"
}
}
],
@@ -30,12 +30,12 @@
"func": "compare_table",
"expected": {
"type": "cloud_file",
- "path": "",
+ "path": "https://101.43.24.67/s/wr7B4GeotNNoeHD",
"dest": "Employee_Roles_and_Ranks_gold.xlsx"
},
"result": {
"type": "vm_file",
- "path": "Desktop/Employee_Roles_and_Ranks.xlsx",
+ "path": "/home/user/Employee_Roles_and_Ranks.xlsx",
"dest": "Employee_Roles_and_Ranks.xlsx"
}
}
diff --git a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
index 14301da..6a9d412 100644
--- a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
+++ b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1ZhGLDKOden_oxzuN2xN9-jNQSHtCX6GE&export=download&authuser=0&confirm=t&uuid=2c097276-a610-4a9f-b6e4-5b54296c1555&at=APZUnTWc7zKPY_ykygn0mO1SAs4s:1703580957447",
- "path": "Desktop/Freeze_row_column.xlsx"
+ "path": "/home/user/Freeze_row_column.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Freeze_row_column.xlsx"
+ "path": "/home/user/Freeze_row_column.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "check_xlsx_freeze",
"result": {
"type": "vm_file",
- "path": "Desktop/Freeze_row_column.xlsx",
+ "path": "/home/user/Freeze_row_column.xlsx",
"dest": "Freeze_row_column.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
index 14251e2..f45c50c 100644
--- a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
+++ b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json
@@ -10,7 +10,7 @@
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=1LHxVvEpLI7kp0Iiy8K74gwkoGiwcLeYP&export=download&authuser=0&confirm=t&uuid=690287ee-d413-46e7-9b01-c56c12e445ff&at=APZUnTVCSd_ajhMGWpEgLHiExfbf:1704199487820",
- "path": "/home/david/Padding_Decimals_In_Formular.xlsx"
+ "path": "/home/user/Padding_Decimals_In_Formular.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "/home/david/Padding_Decimals_In_Formular.xlsx"
+ "path": "/home/user/Padding_Decimals_In_Formular.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "compare_table",
"result": {
"type": "vm_file",
- "path": "/home/david/Padding_Decimals_In_Formular_gold.xlsx",
+ "path": "/home/user/Padding_Decimals_In_Formular_gold.xlsx",
"dest": "Padding_Decimals_In_Formular.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
index d99f170..6ed124f 100644
--- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
+++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json
@@ -10,7 +10,7 @@
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=1uT0axjo9lwkKu6hYVnsAL2FCrdH0DLUv&export=download&authuser=0&confirm=t&uuid=e7da6304-9c7a-4862-8a30-9f2284b843da&at=APZUnTVNHThpAZJmF6IuPckFvslw:1704187618838",
- "path": "/home/david/Set_Decimal_Separator_Dot.xlsx"
+ "path": "/home/user/Set_Decimal_Separator_Dot.xlsx"
}
]
}
@@ -24,7 +24,7 @@
"func": "check_libre_locale",
"result": {
"type": "vm_file",
- "path": "/home/david/.config/libreoffice/4/user/registrymodifications.xcu",
+ "path": "/home/user/.config/libreoffice/4/user/registrymodifications.xcu",
"dest": "registrymodifications.xcu"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json b/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json
index ff3d191..3704a7b 100644
--- a/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json
+++ b/evaluation_examples/examples/libreoffice_calc/aa3a8974-2e85-438b-b29e-a64df44deb4b.json
@@ -10,7 +10,7 @@
"file": [
{
"url": "https://drive.usercontent.google.com/download?id=1O4bw7jEsVdFGeGeSX8hDjsvIbozV38sd&export=download&authuser=0&confirm=t&uuid=b6ceade0-e9c3-47bf-8c40-fef77b5ea1f1&at=APZUnTUUYaEx0Y_lAESeK1DfQZw6:1704179724348",
- "path": "/home/david/Resize_Cells_Fit_Page.xlsx"
+ "path": "/home/user/Resize_Cells_Fit_Page.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "/home/david/Resize_Cells_Fit_Page.xlsx"
+ "path": "/home/user/Resize_Cells_Fit_Page.xlsx"
}
}
],
@@ -30,7 +30,7 @@
"func": "check_pdf_pages",
"result": {
"type": "vm_file",
- "path": "/home/david/Resize_Cells_Fit_Page.xlsx",
+ "path": "/home/user/Resize_Cells_Fit_Page.xlsx",
"dest": "Resize_Cells_Fit_Page.xlsx"
},
"expected": {
diff --git a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
index a9cae16..8d96854 100644
--- a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
+++ b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1rwhniaClEkF8XFzdfaNUA6GmAiy4syMZ&export=download&authuser=0&confirm=t&uuid=6fdd5b04-85f4-45e1-ad74-368f8f2a82ab&at=APZUnTUP-JxPxLfNls6jXWghblQ5:1701766091851",
- "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx"
+ "path": "/home/user/Quarterly_Product_Sales_by_Zone.xlsx"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
- "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx"
+ "path": "/home/user/Quarterly_Product_Sales_by_Zone.xlsx"
}
}
],
@@ -35,7 +35,7 @@
},
"result": {
"type": "vm_file",
- "path": "Desktop/Quarterly_Product_Sales_by_Zone.xlsx",
+ "path": "/home/user/Quarterly_Product_Sales_by_Zone.xlsx",
"dest": "Quarterly_Product_Sales_by_Zone.xlsx"
}
}
diff --git a/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json b/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json
index ea4b832..969b193 100644
--- a/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json
+++ b/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json
@@ -1,12 +1,70 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "thunderbird",
- "instruction": "Could you help me back up all the email files in my profile to PROFILE_DIR?",
+ "instruction": "Could you help me back up all the email files in my profile to ~/email.bak? Please save them in eml format.",
"source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird",
- "config": [],
- "trajectory": "trajectories/",
+ "config": [
+ {
+ "type": "download",
+ "parameters": {
+ "files": [
+ {
+ "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+ "path": "/home/user/thunderbird-profile.tar.gz"
+ }
+ ]
+ }
+ },
+ {
+ "type": "execute",
+ "parameters": {
+ "command": [
+ "tar",
+ "-xzv",
+ "--recursive-unlink",
+ "-f",
+ "/home/user/thunderbird-profile.tar.gz",
+ "-C",
+ "/home/user/"
+ ]
+ }
+ },
+ {
+ "type": "launch",
+ "parameters": {
+ "command": [
+ "/usr/bin/thunderbird"
+ ]
+ }
+ }
+ ],
+ "trajectory": "trajectories/06fe7178-4491-4589-810f-2e2bc9502122",
"related_apps": [
"thunderbird"
],
- "evaluator": "evaluation_dir"
+ "evaluator": {
+ "postconfig": [
+ {
+ "type": "command",
+ "parameters": {
+ "command": ["ls", "-R", "/home/user/emails.bak"],
+ "stdout": "emails.bak.ls"
+ }
+ }
+ ],
+ "func": "check_list",
+ "result": {
+ "type": "cache_file",
+ "path": "emails.bak.ls"
+ },
+ "expected": {
+ "type": "rule",
+ "rules": {
+ "expect": [
+ "歡迎使用新的 Outlook.com 帳戶.*\\.eml",
+ "A Test E-mail.*\\.eml"
+ ]
+ }
+ }
+ }
}
diff --git a/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json
index ce68b66..be50b20 100644
--- a/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json
+++ b/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json
@@ -3,10 +3,57 @@
"snapshot": "thunderbird",
"instruction": "Could you help me open up the Thunderbird profile manager utility?",
"source": "https://www.quora.com/How-do-I-open-a-Thunderbird-profile-manager-utility",
- "config": [],
- "trajectory": "trajectories/",
+ "config": [
+ {
+ "type": "download",
+ "parameters": {
+ "files": [
+ {
+ "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
+ "path": "/home/user/thunderbird-profile.tar.gz"
+ }
+ ]
+ }
+ },
+ {
+ "type": "execute",
+ "parameters": {
+ "command": [
+ "tar",
+ "-xzv",
+ "--recursive-unlink",
+ "-f",
+ "/home/user/thunderbird-profile.tar.gz",
+ "-C",
+ "/home/user/"
+ ]
+ }
+ },
+ {
+ "type": "launch",
+ "parameters": {
+ "command": [
+ "/usr/bin/thunderbird"
+ ]
+ }
+ }
+ ],
+ "trajectory": "trajectories/12086550-11c0-466b-b367-1d9e75b3910e",
"related_apps": [
"thunderbird"
],
- "evaluator": "evaluation_dir"
+ "evaluator": {
+ "result": {
+ "type": "accessibility_tree"
+ },
+ "expected": {
+ "type": "rule",
+ "rules": {
+ "selectors": [
+ "application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]"
+ ]
+ }
+ },
+ "func": "check_accessibility_tree"
+ }
}
diff --git a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
index 449fa26..142cd9f 100644
--- a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
+++ b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
- "path": "/home/david/thunderbird-profile.tar.gz"
+ "path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
@@ -23,9 +23,9 @@
"-xzv",
"--recursive-unlink",
"-f",
- "/home/david/thunderbird-profile.tar.gz",
+ "/home/user/thunderbird-profile.tar.gz",
"-C",
- "/home/david/"
+ "/home/user/"
]
}
},
@@ -50,7 +50,7 @@
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
- "path": "/home/david/firefox_decrypt.py"
+ "path": "/home/user/firefox_decrypt.py"
}
]
}
@@ -60,8 +60,8 @@
"parameters": {
"command": [
"python3",
- "/home/david/firefox_decrypt.py",
- "/home/david/.thunderbird",
+ "/home/user/firefox_decrypt.py",
+ "/home/user/.thunderbird",
"-n",
"-c",
"2",
diff --git a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
index 7e977de..a50953e 100644
--- a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
+++ b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
- "path": "/home/david/thunderbird-profile.tar.gz"
+ "path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
@@ -23,9 +23,9 @@
"-xzv",
"--recursive-unlink",
"-f",
- "/home/david/thunderbird-profile.tar.gz",
+ "/home/user/thunderbird-profile.tar.gz",
"-C",
- "/home/david/"
+ "/home/user/"
]
}
},
@@ -50,7 +50,7 @@
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py",
- "path": "/home/david/firefox_decrypt.py"
+ "path": "/home/user/firefox_decrypt.py"
}
]
}
@@ -60,8 +60,8 @@
"parameters": {
"command": [
"python3",
- "/home/david/firefox_decrypt.py",
- "/home/david/.thunderbird",
+ "/home/user/firefox_decrypt.py",
+ "/home/user/.thunderbird",
"-n",
"-c",
"2",
diff --git a/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json b/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json
index ce982b8..bf63054 100644
--- a/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json
+++ b/evaluation_examples/examples/vs_code/eabc805a-bfcf-4460-b250-ac92135819f6.json
@@ -9,5 +9,14 @@
"vscode"
],
"evaluator": {
+ "func": "compare_answer",
+ "expected": {
+ "type": "string",
+ "string": "ms-python.python\n"
+ },
+ "result": {
+ "type": "command_line",
+ "command": "code --list-extensions | grep ms-python.python"
+ }
}
}
diff --git a/main.py b/main.py
index ce1c0bd..f062e85 100644
--- a/main.py
+++ b/main.py
@@ -44,9 +44,9 @@ def human_agent():
Runs the Gym environment with human input.
"""
- with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f:
+ with open("evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json", "r") as f:
example = json.load(f)
- example["snapshot"] = "base_setup3"
+ example["snapshot"] = "Snapshot 11"
env = DesktopEnv(
path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx",
diff --git a/requirements.txt b/requirements.txt
index 50a95a2..558098b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,4 +23,8 @@ python-docx
python-pptx
pypdf
PyGetWindow
+rapidfuzz
+pyacoustid
+opencv-python
+ImageHash
scikit-image