From df8be173945b78aceb95fe5e37167a7e79bd020e Mon Sep 17 00:00:00 2001 From: David Chang Date: Mon, 8 Jan 2024 23:14:53 +0800 Subject: [PATCH 1/5] ver Jan8th trying to going on setting up thunderbird, but nothing done by now --- desktop_env/evaluators/metrics/general.py | 2 +- desktop_env/evaluators/metrics/thunderbird.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 desktop_env/evaluators/metrics/thunderbird.py diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 37a3bc0..2fb2af7 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -22,7 +22,7 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: unexpect_metric = True with open(result) as f: reader = csv.DictReader(f) - + for rcd in reader: for i, r in enumerate(rules.get("expect", [])): expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd) diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py new file mode 100644 index 0000000..162a8ae --- /dev/null +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -0,0 +1,34 @@ +#from playwright.sync_api import sync_playwright, Browser +#from marionette_driver.marionette import Marionette +#import marionette +import pyatspi + +if __name__ == "__main__": + #with sync_playwright() as plwr: + #while True: + ##try: + #thunderbird: Browser = plwr.firefox.connect("http://127.0.0.1:6000", timeout=60) + #break + ##except: + ##pass + #for ctx in thunderbird.contexts: + #for p in ctx.pages: + #print(p.url) + + #thunderbird = Marionette() + #print(thunderbird.start_session()) + #print(thunderbird.chrome_window_handles) + #print(thunderbird.window_handles) + #print(thunderbird.current_chrome_window_handle) + #thunderbird.set_context(Marionette.CONTEXT_CONTENT) + #print(thunderbird.current_window_handle) + #thunderbird.switch_to_window(thunderbird.chrome_window_handles[0]) + #thunderbird.switch_to_default_content() + #thunderbird.switch_to_frame() + #print(thunderbird.get_url()) + #print(thunderbird.get_window_type()) + #thunderbird.fullscreen() + #print(thunderbird.close()) + + #registry = pyatspi.Registry.get_default() + #registry From ec3bc3079fb0b79a73c111252f820b9d3511a6f5 Mon Sep 17 00:00:00 2001 From: David Chang Date: Tue, 9 Jan 2024 23:01:23 +0800 Subject: [PATCH 2/5] ver Jan9th implemented get_accessibility_tree to convert the accessibility tree from pyatspi to an xml format similar to view hierarchy and HTML havn't merged into server/main.py tested the AT structure of main window of Thunderbird --- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/getters/misc.py | 160 +++++++++++++++++- desktop_env/evaluators/metrics/thunderbird.py | 21 ++- 3 files changed, 178 insertions(+), 5 deletions(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 1fd3c02..ed68081 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,3 +1,3 @@ from .file import get_cloud_file, get_vm_file, get_cache_file -from .misc import get_rule +from .misc import get_rule, get_desktop_path, get_wallpaper, get_accessibility_tree from .vlc import get_vlc_playing_info diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index f66322b..51c0732 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -1,10 +1,20 @@ -from typing import TypeVar +from typing import TypeVar, Any +from typing import Dict, List import platform import subprocess import ctypes import os +import pyatspi +from pyatspi import Accessible, StateType +from pyatspi import Component, Document +from pyatspi import Text as ATText +from pyatspi import Value as ATValue +from pyatspi import Action as ATAction +import lxml.etree +from lxml.etree import _Element + import logging logger = logging.getLogger("desktopenv.getters.misc") @@ -16,7 +26,7 @@ def get_rule(env, config: R) -> R: return config["rules"] -def get_desktop_path(): +def get_desktop_path(*args): username = os.getlogin() # Get the current username if platform.system() == "Windows": return os.path.join("C:", "Users", username, "Desktop") @@ -28,7 +38,7 @@ def get_desktop_path(): raise Exception("Unsupported operating system") -def get_wallpaper(): +def get_wallpaper(*args): def get_wallpaper_windows(): SPI_GETDESKWALLPAPER = 0x73 MAX_PATH = 260 @@ -64,3 +74,147 @@ def get_wallpaper(): return get_wallpaper_linux() else: return "Unsupported OS" + +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } +def _create_node(node: Accessible) -> _Element: + attribute_dict: Dict[str, Any] = {"name": node.name} + + # States {{{ # + states: List[StateType] = node.getState().get_states() + for st in states: + state_name: str = StateType._enum_lookup[st] + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["st"] + , state_name.split("_", maxsplit=1)[1].lower() + ) + ] = "true" + # }}} States # + + # Attributes {{{ # + attributes: List[str] = node.getAttributes() + for attrbt in attributes: + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["attr"] + , attribute_name + ) + ] = attribute_value + # }}} Attributes # + + # Component {{{ # + try: + component: Component = node.queryComponent() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN)) + attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW)) + attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT)) + attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize()) + # }}} Component # + + # Document {{{ # + try: + document: Document = node.queryDocument() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale() + attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount()) + attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber()) + for attrbt in document.getAttributes(): + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["docattr"] + , attribute_name + ) + ] = attribute_value + # }}} Document # + + # Text {{{ # + try: + text_obj: ATText = node.queryText() + except NotImplementedError: + pass + else: + # only text shown on current screen is available + #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount) + text: str = text_obj.getText(0, text_obj.characterCount) + # }}} Text # + + # Selection {{{ # + try: + node.querySelection() + except NotImplementedError: + pass + else: + attribute_dict["selection"] = "true" + # }}} Selection # + + # Value {{{ # + try: + value: ATValue = node.queryValue() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue) + attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue) + attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue) + attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement) + # }}} Value # + + # Action {{{ # + try: + action: ATAction = node.queryAction() + except NotImplementedError: + pass + else: + for i in range(action.nActions): + action_name: str = action.getName(i).replace(" ", "-") + attribute_dict[ "{{{:}}}{:}_desc"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getDescription(i) + attribute_dict[ "{{{:}}}{:}_kb"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getKeyBinding(i) + # }}} Action # + + xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-") + , attrib=attribute_dict + , nsmap=_accessibility_ns_map + ) + if "text" in locals() and len(text)>0: + xml_node.text = text + for ch in node: + xml_node.append(_create_node(ch)) + return xml_node + +def get_accessibility_tree(*args) -> _Element: + desktop: Accessible = pyatspi.Registry.getDesktop(0) + desktop_xml: _Element = _create_node(desktop) + return desktop_xml + +if __name__ == "__main__": + import sys + with open(sys.argv[1], "w") as f: + f.write( lxml.etree.tostring( get_accessibility_tree() + , encoding="unicode" + , pretty_print=True + ) + ) diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py index 162a8ae..581b8d1 100644 --- a/desktop_env/evaluators/metrics/thunderbird.py +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -1,7 +1,13 @@ #from playwright.sync_api import sync_playwright, Browser #from marionette_driver.marionette import Marionette #import marionette -import pyatspi +#import pyatspi + +import lxml.etree +from lxml.cssselect import CSSSelector +from lxml.etree import _Element + +from typing import List if __name__ == "__main__": #with sync_playwright() as plwr: @@ -32,3 +38,16 @@ if __name__ == "__main__": #registry = pyatspi.Registry.get_default() #registry + + #xml = "../../任务数据/Thunderbird/vertical-card-view.xml" + xml = "../../任务数据/Thunderbird/vertical-table-view.xml" + at: _Element = lxml.etree.parse(xml) + + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + print(len(elements)) + for elm in elements: + print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True)) From cf5d480f440cfa23aae9e51cc2b7960fa49cc82f Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 10 Jan 2024 17:36:59 +0800 Subject: [PATCH 3/5] ver Jan10th new Thunderbird task config --- desktop_env/controllers/python.py | 25 ++- desktop_env/envs/desktop_env.py | 2 +- desktop_env/evaluators/getters/misc.py | 177 +++--------------- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/general.py | 65 +++++++ desktop_env/server/main.py | 153 ++++++++++++++- .../12086550-11c0-466b-b367-1d9e75b3910e.json | 53 +++++- .../7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json | 12 +- .../bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json | 12 +- main.py | 6 +- 10 files changed, 325 insertions(+), 182 deletions(-) diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index 3f077fa..81a070b 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict +from typing import Any, Dict, Optional import requests from desktop_env.envs.actions import KEYBOARD_KEYS @@ -22,6 +22,15 @@ class PythonController: logger.error("Failed to get screenshot. Status code: %d", response.status_code) return None + def get_accessibility_tree(self) -> Optional[str]: + + response: requests.Response = requests.get(self.http_server + "/accessibility") + if response.status_code == 200: + return response.json()["AT"] + else: + logger.error("Failed to get accessibility tree. Status code: %d", response.status_code) + return None + def get_file(self, file_path: str): """ Gets a file from the server. @@ -65,7 +74,7 @@ class PythonController: if action_type == "MOVE_TO": if parameters == {} or None: - self.execute_python_command(f"pyautogui.moveTo()") + self.execute_python_command("pyautogui.moveTo()") elif "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] @@ -75,7 +84,7 @@ class PythonController: elif action_type == "CLICK": if parameters == {} or None: - self.execute_python_command(f"pyautogui.click()") + self.execute_python_command("pyautogui.click()") elif "button" in parameters and "x" in parameters and "y" in parameters: button = parameters["button"] x = parameters["x"] @@ -106,7 +115,7 @@ class PythonController: elif action_type == "MOUSE_DOWN": if parameters == {} or None: - self.execute_python_command(f"pyautogui.mouseDown()") + self.execute_python_command("pyautogui.mouseDown()") elif "button" in parameters: button = parameters["button"] self.execute_python_command(f"pyautogui.mouseDown(button='{button}')") @@ -115,7 +124,7 @@ class PythonController: elif action_type == "MOUSE_UP": if parameters == {} or None: - self.execute_python_command(f"pyautogui.mouseUp()") + self.execute_python_command("pyautogui.mouseUp()") elif "button" in parameters: button = parameters["button"] self.execute_python_command(f"pyautogui.mouseUp(button='{button}')") @@ -124,7 +133,7 @@ class PythonController: elif action_type == "RIGHT_CLICK": if parameters == {} or None: - self.execute_python_command(f"pyautogui.rightClick()") + self.execute_python_command("pyautogui.rightClick()") elif "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] @@ -134,7 +143,7 @@ class PythonController: elif action_type == "DOUBLE_CLICK": if parameters == {} or None: - self.execute_python_command(f"pyautogui.doubleClick()") + self.execute_python_command("pyautogui.doubleClick()") elif "x" in parameters and "y" in parameters: x = parameters["x"] y = parameters["y"] @@ -200,7 +209,7 @@ class PythonController: raise Exception(f"Unknown parameters: {parameters}") keys = parameters["keys"] if not isinstance(keys, list): - raise Exception(f"Keys must be a list of keys") + raise Exception("Keys must be a list of keys") for key in keys: if key.lower() not in KEYBOARD_KEYS: raise Exception(f"Key must be one of {KEYBOARD_KEYS}") diff --git a/desktop_env/envs/desktop_env.py b/desktop_env/envs/desktop_env.py index b78e4b5..e256f7c 100644 --- a/desktop_env/envs/desktop_env.py +++ b/desktop_env/envs/desktop_env.py @@ -223,7 +223,7 @@ class DesktopEnv(gym.Env): Evaluate whether the task is successfully completed. """ - self.setup_controller.setup(self.evaluator["postconfig"]) + self.setup_controller.setup(self.evaluator.get("postconfig", [])) result_state = self.result_getter(self, self.evaluator["result"]) expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \ diff --git a/desktop_env/evaluators/getters/misc.py b/desktop_env/evaluators/getters/misc.py index 51c0732..d176b47 100644 --- a/desktop_env/evaluators/getters/misc.py +++ b/desktop_env/evaluators/getters/misc.py @@ -1,19 +1,19 @@ -from typing import TypeVar, Any -from typing import Dict, List +from typing import TypeVar +#from typing import Dict, List import platform import subprocess import ctypes import os -import pyatspi -from pyatspi import Accessible, StateType -from pyatspi import Component, Document -from pyatspi import Text as ATText -from pyatspi import Value as ATValue -from pyatspi import Action as ATAction -import lxml.etree -from lxml.etree import _Element +#import pyatspi +#from pyatspi import Accessible, StateType +#from pyatspi import Component, Document +#from pyatspi import Text as ATText +#from pyatspi import Value as ATValue +#from pyatspi import Action as ATAction +#import lxml.etree +#from lxml.etree import _Element import logging logger = logging.getLogger("desktopenv.getters.misc") @@ -75,146 +75,21 @@ def get_wallpaper(*args): else: return "Unsupported OS" -_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" - , "attr": "uri:deskat:attributes.at-spi.gnome.org" - , "cp": "uri:deskat:component.at-spi.gnome.org" - , "doc": "uri:deskat:document.at-spi.gnome.org" - , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" - , "txt": "uri:deskat:text.at-spi.gnome.org" - , "val": "uri:deskat:value.at-spi.gnome.org" - , "act": "uri:deskat:action.at-spi.gnome.org" - } -def _create_node(node: Accessible) -> _Element: - attribute_dict: Dict[str, Any] = {"name": node.name} +#def get_accessibility_tree(*args) -> _Element: + #desktop: Accessible = pyatspi.Registry.getDesktop(0) + #desktop_xml: _Element = _create_node(desktop) + #return desktop_xml - # States {{{ # - states: List[StateType] = node.getState().get_states() - for st in states: - state_name: str = StateType._enum_lookup[st] - attribute_dict[ "{{{:}}}{:}"\ - .format( _accessibility_ns_map["st"] - , state_name.split("_", maxsplit=1)[1].lower() - ) - ] = "true" - # }}} States # +def get_accessibility_tree(env, *args) -> str: + accessibility_tree: str = env.controller.get_accessibility_tree() + logger.debug("AT@eval: %s", accessibility_tree) + return accessibility_tree - # Attributes {{{ # - attributes: List[str] = node.getAttributes() - for attrbt in attributes: - attribute_name: str - attribute_value: str - attribute_name, attribute_value = attrbt.split(":", maxsplit=1) - attribute_dict[ "{{{:}}}{:}"\ - .format( _accessibility_ns_map["attr"] - , attribute_name - ) - ] = attribute_value - # }}} Attributes # - - # Component {{{ # - try: - component: Component = node.queryComponent() - except NotImplementedError: - pass - else: - attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN)) - attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW)) - attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT)) - attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize()) - # }}} Component # - - # Document {{{ # - try: - document: Document = node.queryDocument() - except NotImplementedError: - pass - else: - attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale() - attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount()) - attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber()) - for attrbt in document.getAttributes(): - attribute_name: str - attribute_value: str - attribute_name, attribute_value = attrbt.split(":", maxsplit=1) - attribute_dict[ "{{{:}}}{:}"\ - .format( _accessibility_ns_map["docattr"] - , attribute_name - ) - ] = attribute_value - # }}} Document # - - # Text {{{ # - try: - text_obj: ATText = node.queryText() - except NotImplementedError: - pass - else: - # only text shown on current screen is available - #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount) - text: str = text_obj.getText(0, text_obj.characterCount) - # }}} Text # - - # Selection {{{ # - try: - node.querySelection() - except NotImplementedError: - pass - else: - attribute_dict["selection"] = "true" - # }}} Selection # - - # Value {{{ # - try: - value: ATValue = node.queryValue() - except NotImplementedError: - pass - else: - attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue) - attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue) - attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue) - attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement) - # }}} Value # - - # Action {{{ # - try: - action: ATAction = node.queryAction() - except NotImplementedError: - pass - else: - for i in range(action.nActions): - action_name: str = action.getName(i).replace(" ", "-") - attribute_dict[ "{{{:}}}{:}_desc"\ - .format( _accessibility_ns_map["act"] - , action_name - ) - ] = action.getDescription(i) - attribute_dict[ "{{{:}}}{:}_kb"\ - .format( _accessibility_ns_map["act"] - , action_name - ) - ] = action.getKeyBinding(i) - # }}} Action # - - xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-") - , attrib=attribute_dict - , nsmap=_accessibility_ns_map - ) - if "text" in locals() and len(text)>0: - xml_node.text = text - for ch in node: - xml_node.append(_create_node(ch)) - return xml_node - -def get_accessibility_tree(*args) -> _Element: - desktop: Accessible = pyatspi.Registry.getDesktop(0) - desktop_xml: _Element = _create_node(desktop) - return desktop_xml - -if __name__ == "__main__": - import sys - with open(sys.argv[1], "w") as f: - f.write( lxml.etree.tostring( get_accessibility_tree() - , encoding="unicode" - , pretty_print=True - ) - ) +#if __name__ == "__main__": + #import sys + #with open(sys.argv[1], "w") as f: + #f.write( lxml.etree.tostring( get_accessibility_tree() + #, encoding="unicode" + #, pretty_print=True + #) + #) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 43a044f..e8f313b 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -6,4 +6,4 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im from .pdf import check_pdf_pages from .libreoffice import check_libre_locale #from .vlc import is_vlc_playing -from .general import check_csv +from .general import check_csv, check_accessibility_tree diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 2fb2af7..8ede586 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -1,5 +1,16 @@ import csv + +import lxml.etree +from lxml.etree import _Element +from lxml.cssselect import CSSSelector + from typing import Dict, List +from typing import Callable, Any +from numbers import Number + +import operator +from rapidfuzz import fuzz +import functools def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float: return all(k in item and item[k]==val for k, val in pattern.items()) @@ -28,3 +39,57 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd) unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", [])) return float(all(expect_metrics) and unexpect_metric) + +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } +def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: + """ + Args: + result (str): XML of GNOME Accessibility Tree + rules (Dict[str, Any]): dict like + { + "selectors": list of str as CSS selectors, will be connected by ", " + to form a composite selector. Only one from `selectors` and + `xpath` is needed. If both are present, `xpath` takes the + priority. + "xpath": str as xpath. Only one from `selectors` and `xpath` is + needed. If both are present, `xpath` takes the priority. + "text": str as the expected text content of the selected element. + "exact": bool specifying whether exact match or fuzzy match should + be performed. defaults to True + } + + Returns: + float + """ + + at: _Element = lxml.etree.fromstring(result) + if "xpath" in rules: + elements: List[_Element] = at.xpath(rules["xpath"], namespaces=_accessibility_ns_map) + elif "selectors" in rules: + selector = CSSSelector(", ".join(rules["selectors"]), namespaces=_accessibility_ns_map) + elements: List[_Element] = selector(at) + else: + raise ValueError("At least one of xpath and selectors is required") + + if len(elements)==0: + return 0. + + if "text" in rules: + match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio + , rules["text"] + ) + match_score: Number = 0 + for elm in elements: + match_score = max(match_score, match_func(elm.text or None)) + else: + match_score = 1. + + return float(match_score) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 26fc2a4..56101b5 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -2,14 +2,26 @@ import os from pathlib import Path import platform import subprocess -import requests -from .pyxcursor import Xcursor + +from pyxcursor import Xcursor # import Xlib.display import pyautogui # from PIL import ImageGrab, Image from PIL import Image +import lxml.etree +from lxml.etree import _Element +import pyatspi +from pyatspi import Accessible, StateType +from pyatspi import Component, Document +from pyatspi import Text as ATText +from pyatspi import Value as ATValue +from pyatspi import Action as ATAction + +import requests from flask import Flask, request, jsonify, send_file -from typing import List + +from typing import List, Dict +from typing import Any app = Flask(__name__) @@ -100,6 +112,141 @@ def capture_screen_with_cursor(): return send_file(file_path, mimetype='image/png') +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + } +def _create_node(node: Accessible) -> _Element: + attribute_dict: Dict[str, Any] = {"name": node.name} + + # States {{{ # + states: List[StateType] = node.getState().get_states() + for st in states: + state_name: str = StateType._enum_lookup[st] + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["st"] + , state_name.split("_", maxsplit=1)[1].lower() + ) + ] = "true" + # }}} States # + + # Attributes {{{ # + attributes: List[str] = node.getAttributes() + for attrbt in attributes: + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["attr"] + , attribute_name + ) + ] = attribute_value + # }}} Attributes # + + # Component {{{ # + try: + component: Component = node.queryComponent() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN)) + attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW)) + attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT)) + attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize()) + # }}} Component # + + # Document {{{ # + try: + document: Document = node.queryDocument() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale() + attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount()) + attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber()) + for attrbt in document.getAttributes(): + attribute_name: str + attribute_value: str + attribute_name, attribute_value = attrbt.split(":", maxsplit=1) + attribute_dict[ "{{{:}}}{:}"\ + .format( _accessibility_ns_map["docattr"] + , attribute_name + ) + ] = attribute_value + # }}} Document # + + # Text {{{ # + try: + text_obj: ATText = node.queryText() + except NotImplementedError: + pass + else: + # only text shown on current screen is available + #attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount) + text: str = text_obj.getText(0, text_obj.characterCount) + # }}} Text # + + # Selection {{{ # + try: + node.querySelection() + except NotImplementedError: + pass + else: + attribute_dict["selection"] = "true" + # }}} Selection # + + # Value {{{ # + try: + value: ATValue = node.queryValue() + except NotImplementedError: + pass + else: + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue) + attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue) + attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue) + attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement) + # }}} Value # + + # Action {{{ # + try: + action: ATAction = node.queryAction() + except NotImplementedError: + pass + else: + for i in range(action.nActions): + action_name: str = action.getName(i).replace(" ", "-") + attribute_dict[ "{{{:}}}{:}_desc"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getDescription(i) + attribute_dict[ "{{{:}}}{:}_kb"\ + .format( _accessibility_ns_map["act"] + , action_name + ) + ] = action.getKeyBinding(i) + # }}} Action # + + xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-") + , attrib=attribute_dict + , nsmap=_accessibility_ns_map + ) + if "text" in locals() and len(text)>0: + xml_node.text = text + for ch in node: + xml_node.append(_create_node(ch)) + return xml_node + +@app.route("/accessibility", methods=["GET"]) +def get_accessibility_tree(): + desktop: Accessible = pyatspi.Registry.getDesktop(0) + desktop_xml: _Element = _create_node(desktop) + return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")}) @app.route('/file', methods=['POST']) def get_file(): diff --git a/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json b/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json index ce68b66..be50b20 100644 --- a/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json +++ b/evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json @@ -3,10 +3,57 @@ "snapshot": "thunderbird", "instruction": "Could you help me open up the Thunderbird profile manager utility?", "source": "https://www.quora.com/How-do-I-open-a-Thunderbird-profile-manager-utility", - "config": [], - "trajectory": "trajectories/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/12086550-11c0-466b-b367-1d9e75b3910e", "related_apps": [ "thunderbird" ], - "evaluator": "evaluation_dir" + "evaluator": { + "result": { + "type": "accessibility_tree" + }, + "expected": { + "type": "rule", + "rules": { + "selectors": [ + "application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]" + ] + } + }, + "func": "check_accessibility_tree" + } } diff --git a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json index 449fa26..142cd9f 100644 --- a/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json +++ b/evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", - "path": "/home/david/thunderbird-profile.tar.gz" + "path": "/home/user/thunderbird-profile.tar.gz" } ] } @@ -23,9 +23,9 @@ "-xzv", "--recursive-unlink", "-f", - "/home/david/thunderbird-profile.tar.gz", + "/home/user/thunderbird-profile.tar.gz", "-C", - "/home/david/" + "/home/user/" ] } }, @@ -50,7 +50,7 @@ "files": [ { "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", - "path": "/home/david/firefox_decrypt.py" + "path": "/home/user/firefox_decrypt.py" } ] } @@ -60,8 +60,8 @@ "parameters": { "command": [ "python3", - "/home/david/firefox_decrypt.py", - "/home/david/.thunderbird", + "/home/user/firefox_decrypt.py", + "/home/user/.thunderbird", "-n", "-c", "2", diff --git a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json index 7e977de..a50953e 100644 --- a/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json +++ b/evaluation_examples/examples/thunderbird/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", - "path": "/home/david/thunderbird-profile.tar.gz" + "path": "/home/user/thunderbird-profile.tar.gz" } ] } @@ -23,9 +23,9 @@ "-xzv", "--recursive-unlink", "-f", - "/home/david/thunderbird-profile.tar.gz", + "/home/user/thunderbird-profile.tar.gz", "-C", - "/home/david/" + "/home/user/" ] } }, @@ -50,7 +50,7 @@ "files": [ { "url": "https://raw.githubusercontent.com/unode/firefox_decrypt/main/firefox_decrypt.py", - "path": "/home/david/firefox_decrypt.py" + "path": "/home/user/firefox_decrypt.py" } ] } @@ -60,8 +60,8 @@ "parameters": { "command": [ "python3", - "/home/david/firefox_decrypt.py", - "/home/david/.thunderbird", + "/home/user/firefox_decrypt.py", + "/home/user/.thunderbird", "-n", "-c", "2", diff --git a/main.py b/main.py index cebb934..b89cf79 100644 --- a/main.py +++ b/main.py @@ -44,11 +44,11 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f: + with open("evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 13" + example["snapshot"] = "Snapshot 9" - env = DesktopEnv( path_to_vm="/home/david/vmware/KUbuntu 64-bit/KUbuntu 64-bit.vmx" + env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" , action_space="computer_13" , task_config=example ) From 1515b05666791f0c42eca881ebb19cb8c962ed3d Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 10 Jan 2024 21:58:29 +0800 Subject: [PATCH 4/5] ver Jan10thv2 a new example config for Thunderbird fixed several bugs --- desktop_env/controllers/setup.py | 3 + desktop_env/evaluators/getters/file.py | 5 +- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/general.py | 34 +++++++++- desktop_env/evaluators/metrics/table.py | 12 ++++ .../06fe7178-4491-4589-810f-2e2bc9502122.json | 66 +++++++++++++++++-- main.py | 4 +- requirements.txt | 1 + 8 files changed, 117 insertions(+), 10 deletions(-) diff --git a/desktop_env/controllers/setup.py b/desktop_env/controllers/setup.py index a960c92..7704866 100644 --- a/desktop_env/controllers/setup.py +++ b/desktop_env/controllers/setup.py @@ -242,6 +242,9 @@ class SetupController: logger.error("An error occurred while trying to send the request: %s", e) traceback.print_exc() + def _command_setup(self, command: List[str], stdout: str = "", stderr: str = ""): + self._execute_setup(command, stdout, stderr) + def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]): # TODO raise NotImplementedError() diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 2c59035..2d77eba 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -1,4 +1,5 @@ from typing import Dict +from typing import Optional import os import requests @@ -27,7 +28,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str: return _path -def get_vm_file(env, config: Dict[str, str]) -> str: +def get_vm_file(env, config: Dict[str, str]) -> Optional[str]: """ Config: path (str): absolute path on the VM to fetch @@ -37,6 +38,8 @@ def get_vm_file(env, config: Dict[str, str]) -> str: _path = os.path.join(env.cache_dir, config["dest"]) file = env.controller.get_file(config["path"]) + if file is None: + return None with open(_path, "wb") as f: f.write(file) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index e8f313b..1beb03e 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -6,4 +6,4 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im from .pdf import check_pdf_pages from .libreoffice import check_libre_locale #from .vlc import is_vlc_playing -from .general import check_csv, check_accessibility_tree +from .general import check_csv, check_accessibility_tree, check_list diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 8ede586..427198c 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -4,13 +4,14 @@ import lxml.etree from lxml.etree import _Element from lxml.cssselect import CSSSelector -from typing import Dict, List +from typing import Dict, List, Pattern from typing import Callable, Any from numbers import Number import operator from rapidfuzz import fuzz import functools +import re def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float: return all(k in item and item[k]==val for k, val in pattern.items()) @@ -37,7 +38,33 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: for rcd in reader: for i, r in enumerate(rules.get("expect", [])): expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd) - unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", [])) + unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", [])) + return float(all(expect_metrics) and unexpect_metric) + +def check_list(result: str, rules: Dict[str, List[str]]) -> float: + """ + Args: + result (str): path to list file + rules (Dict[str, List[str]]): dict like + { + "expect": list of str as regexes + "unexpect": list of str as regexes + } + + Returns: + float + """ + + expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])] + unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])] + + expect_metrics = [False] * len(expect_patterns) + unexpect_metric = True + with open(result) as f: + for l in f: + for i, r in enumerate(expect_patterns): + expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None) + unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns) return float(all(expect_metrics) and unexpect_metric) _accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" @@ -93,3 +120,6 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: match_score = 1. return float(match_score) + +#def check_existence(result: str, *args) -> float: + #return 1. - (result is None) diff --git a/desktop_env/evaluators/metrics/table.py b/desktop_env/evaluators/metrics/table.py index 34ef0b0..25b55f3 100644 --- a/desktop_env/evaluators/metrics/table.py +++ b/desktop_env/evaluators/metrics/table.py @@ -31,6 +31,9 @@ def compare_table(actual: str, expected: str, **options) -> float: float: the score """ + if actual is None: + return 0. + df1 = pd.read_excel(expected) df2 = pd.read_excel(actual) metric: bool = df1.equals(df2) @@ -71,6 +74,9 @@ def compare_table(actual: str, expected: str, **options) -> float: return float(metric) def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: + if result is None: + return 0. + # workbook: Workbook = openpyxl.load_workbook(filename=result) workbook = pd.ExcelFile(result) worksheet_names: List[str] = workbook.sheet_names @@ -109,10 +115,16 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: return float(passes) def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float: + if result is None: + return 0. + worksheet: Worksheet = openpyxl.load_workbook(filename=result).active return float(worksheet.freeze_panes == rules["position"]) def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float: + if result is None: + return 0. + worksheet = openpyxl.load_workbook(filename=result).active zoom_scale: Number = worksheet.sheet_view.zoomScale or 100. return float( getattr(operator, rules["relation"])( zoom_scale diff --git a/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json b/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json index ea4b832..969b193 100644 --- a/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json +++ b/evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json @@ -1,12 +1,70 @@ { "id": "06fe7178-4491-4589-810f-2e2bc9502122", "snapshot": "thunderbird", - "instruction": "Could you help me back up all the email files in my profile to PROFILE_DIR?", + "instruction": "Could you help me back up all the email files in my profile to ~/email.bak? Please save them in eml format.", "source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird", - "config": [], - "trajectory": "trajectories/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/06fe7178-4491-4589-810f-2e2bc9502122", "related_apps": [ "thunderbird" ], - "evaluator": "evaluation_dir" + "evaluator": { + "postconfig": [ + { + "type": "command", + "parameters": { + "command": ["ls", "-R", "/home/user/emails.bak"], + "stdout": "emails.bak.ls" + } + } + ], + "func": "check_list", + "result": { + "type": "cache_file", + "path": "emails.bak.ls" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + "歡迎使用新的 Outlook.com 帳戶.*\\.eml", + "A Test E-mail.*\\.eml" + ] + } + } + } } diff --git a/main.py b/main.py index b89cf79..b128766 100644 --- a/main.py +++ b/main.py @@ -44,9 +44,9 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json", "r") as f: + with open("evaluation_examples/examples/thunderbird/06fe7178-4491-4589-810f-2e2bc9502122.json", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 9" + example["snapshot"] = "Snapshot 11" env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" , action_space="computer_13" diff --git a/requirements.txt b/requirements.txt index d97aedd..4898f1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,3 +23,4 @@ python-docx python-pptx pypdf PyGetWindow +rapidfuzz From 18cc1fc52cb0e6d8e52055de17bb29637274bbde Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 10 Jan 2024 22:23:48 +0800 Subject: [PATCH 5/5] ver Jan10thv3 minor fixes --- desktop_env/evaluators/getters/__init__.py | 2 +- requirements.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 5590650..a3fe8fe 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -1,3 +1,3 @@ from .file import get_cloud_file, get_vm_file, get_cache_file -from .misc import get_rule, get_desktop_path, get_wallpaper, get_accessibility_tree +from .misc import get_rule, get_accessibility_tree from .vlc import get_vlc_playing_info, get_vlc_config diff --git a/requirements.txt b/requirements.txt index 4898f1e..49268ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,6 @@ python-pptx pypdf PyGetWindow rapidfuzz +pyacoustid +opencv-python +ImageHash