diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 7428eb2..7a44195 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -7,4 +7,4 @@ from .pdf import check_pdf_pages from .libreoffice import check_libre_locale from .vlc import is_vlc_playing, is_vlc_recordings_folder from .general import check_csv, check_accessibility_tree, check_list -from .thunderbird import check_thunderbird_prefs +from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 6f44d83..7acf23f 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -13,8 +13,10 @@ from rapidfuzz import fuzz import functools import re -def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool: - return all(k in item and item[k]==val for k, val in pattern.items()) +from .utils import _match_record + +#def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool: + #return all(k in item and item[k]==val for k, val in pattern.items()) def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ @@ -30,6 +32,9 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: float """ + if result is None: + return 0. + expect_metrics = [False] * len(rules.get("expect", [])) unexpect_metric = True with open(result) as f: @@ -55,6 +60,9 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float: float """ + if result is None: + return 0. + expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])] unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])] diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py index b386243..b202d94 100644 --- a/desktop_env/evaluators/metrics/thunderbird.py +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -1,10 +1,11 @@ -from typing import List, Pattern, Dict, Match -from typing import Union, Any, Optional, Iterable, TypeVar +from typing import List, Pattern, Dict, Match, Tuple +from typing import Union, Any, Optional, Iterable, TypeVar, Callable import re import functools import operator import json +from .utils import _match_record import logging logger = logging.getLogger("desktopenv.metric.thunderbird") @@ -82,20 +83,128 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any return float(all(expect_metrics.values()) and unexpect_metric) +_value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\") +#_condition_pattern: Pattern[str] = re.compile(r'(?PAND|OR) \((?P[\w ]+),(?P[\w ' + '\'' + r']+),(?:"(?P(?:[^"]|\")+)"|(?P[^)]+))\)') +_condition_pattern: Pattern[str] = re.compile(r'(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)') +def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: + """ + Args: + result (str): path to filter def file + rules (Dict[str, List[Dict[str, str]]]): dict like + { + "expect": [{key: value}] + "unexpect": [{key: value}] + } + + Returns: + float + """ + + if result is None: + return 0. + + # read filter def file + # a filter: + # { + # "name": "Name", + # "enabled": "yes" | "no", + # "type": "17", + # "action": "Move to folder" | ..., + # "actionValue": ..., + # "condition": [...] + # } + filters: List[Dict[str, Union[str, List[str]]]] = [] + with open(result) as f: + for l in f: + if l.startswith("name="): + filter_: Dict[str, Union[str, List[str]]] = {} + filter_["name"] = _value_processor(l[6:-2]) + elif l.startswith("enabled="): + filter_["enabled"] = _value_processor(l[9:-2]) + elif l.startswith("type="): + filter_["type"] = _value_processor(l[6:-2]) + elif l.startswith("action="): + filter_["action"] = _value_processor(l[8:-2]) + elif l.startswith("actionValue="): + filter_["actionValue"] = _value_processor(l[13:-2]) + elif l.startswith("condition="): + condition_str: str = _value_processor(l[11:-2]) + logger.debug("FILTER CONDITION: %s", condition_str) + + conditions: List[str] =\ + _condition_pattern.findall(condition_str) + logger.debug("FILTER CONDITIONS: %s", repr(conditions)) + + filter_["condition"] = conditions + logger.debug("FILTER %s", repr(filter_)) + filters.append(filter_) + + expect_metrics = [False] * len(rules.get("expect", [])) + unexpect_metric = True + for flt in filters: + for i, r in enumerate(rules.get("expect", [])): + expect_metrics[i] = expect_metrics[i] or _match_record(r, flt) + unexpect_metric = unexpect_metric and not any(_match_record(r, flt) for r in rules.get("unexpect", [])) + return float(all(expect_metrics) and unexpect_metric) + if __name__ == "__main__": import lxml.etree from lxml.cssselect import CSSSelector from lxml.etree import _Element #xml = "../../任务数据/Thunderbird/vertical-card-view.xml" - xml = "../../任务数据/Thunderbird/vertical-table-view.xml" - at: _Element = lxml.etree.parse(xml) + #xml = "../../任务数据/Thunderbird/vertical-table-view.xml" + #at: _Element = lxml.etree.parse(xml) #elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header - elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header - print(len(elements)) - for elm in elements: - print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True)) + #elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header + #print(len(elements)) + #for elm in elements: + #print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True)) + + import datetime + import os + import sys + + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S") + + file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str))) + debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str))) + stdout_handler = logging.StreamHandler(sys.stdout) + sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str))) + + file_handler.setLevel(logging.INFO) + debug_handler.setLevel(logging.DEBUG) + stdout_handler.setLevel(logging.INFO) + sdebug_handler.setLevel(logging.DEBUG) + + formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s") + file_handler.setFormatter(formatter) + debug_handler.setFormatter(formatter) + stdout_handler.setFormatter(formatter) + sdebug_handler.setFormatter(formatter) + + stdout_handler.addFilter(logging.Filter("desktopenv")) + sdebug_handler.addFilter(logging.Filter("desktopenv")) + + logger.addHandler(file_handler) + logger.addHandler(debug_handler) + logger.addHandler(stdout_handler) + logger.addHandler(sdebug_handler) + + print( check_thunderbird_filter( "../../任务数据/Thunderbird/msgFilterRules.dat" + , { "expect": [ { "enabled": "yes" + , "action": "Move to folder" + , "actionValue": "mailbox://nobody@Local%20Folders/Promotions" + , "condition": ["AND (subject,contains,discount)"] + } + ] + } + ) + ) diff --git a/desktop_env/evaluators/metrics/utils.py b/desktop_env/evaluators/metrics/utils.py index 93bc778..6f20d30 100644 --- a/desktop_env/evaluators/metrics/utils.py +++ b/desktop_env/evaluators/metrics/utils.py @@ -125,6 +125,9 @@ def load_charts(xlsx_file: Workbook, **options) -> Dict[str, Any]: chart_set[series] = info return chart_set +def _match_record(pattern: Dict[str, Any], item: Dict[str, Any]) -> bool: + return all(k in item and item[k]==val for k, val in pattern.items()) + if __name__ == "__main__": path1 = "../../../../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold_line_scatter.xlsx" workbook1: Workbook = openpyxl.load_workbook(filename=path1) diff --git a/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json new file mode 100644 index 0000000..3dd986a --- /dev/null +++ b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json @@ -0,0 +1,77 @@ +{ + "id": "e1e75309-3ddb-4d09-92ec-de869c928143", + "snapshot": "thunderbird", + "instruction": "Create a local folder called \"Promotions\" and create a filter to auto move the inbox emails whose subject contains “discount” to the new folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/e1e75309-3ddb-4d09-92ec-de869c928143", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "command", + "parameters": { + "command": ["wmctrl", "-Fc", "Message Filters"], + "until": { + "returncode": 1 + } + } + } + ], + "func": "check_thunderbird_filter", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat", + "dest": "msgFilterRules.dat" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + { + "enabled": "yes", + "action": "Move to folder", + "actionValue": "mailbox://nobody@Local%20Folders/Promotions", + "condition": ["AND (subject,contains,discount)"] + } + ] + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup new file mode 100644 index 0000000..66af75d --- /dev/null +++ b/evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup @@ -0,0 +1,55 @@ +{ + "id": "e1e75309-3ddb-4d09-92ec-de869c928143", + "snapshot": "thunderbird", + "instruction": "Create a local folder called \"Promotions\" and create a filter to auto move the inbox emails whose subject contains “discount” to the new folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + } + ], + "trajectory": "trajectories/e1e75309-3ddb-4d09-92ec-de869c928143", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "command", + "parameters": { + "command": ["wmctrl", "-Fc", "Message Filters"], + "until": { + "returncode": 1 + } + } + } + ], + "func": "check_thunderbird_filter", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat", + "dest": "msgFilterRules.dat" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + { + "enabled": "yes", + "action": "Move to folder", + "actionValue": "mailbox://nobody@Local%20Folders/Promotions", + "condition": ["AND (subject,contains,discount)"] + } + ] + } + } + } +} diff --git a/main.py b/main.py index b4d1ef8..5da8091 100644 --- a/main.py +++ b/main.py @@ -44,9 +44,9 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup", "r") as f: + with open("evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 17" + example["snapshot"] = "Snapshot 18" env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" , action_space="computer_13"