lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan11thv2

Browse Source 
a new Thunderbird example w.r.t. email filter
This commit is contained in:
David Chang
2024-01-11 22:43:38 +08:00
parent 27eaf2f5d5
commit 3c04872dcf
7 changed files with 265 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -7,4 +7,4 @@ from .pdf import check_pdf_pages
from .libreoffice import check_libre_locale
from .vlc import is_vlc_playing, is_vlc_recordings_folder
from .general import check_csv, check_accessibility_tree, check_list
from .thunderbird import check_thunderbird_prefs
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter

12
desktop_env/evaluators/metrics/general.py
View File

@@ -13,8 +13,10 @@ from rapidfuzz import fuzz
import functools
import re
def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool:
return all(k in item and item[k]==val for k, val in pattern.items())
from .utils import _match_record
#def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> bool:
#return all(k in item and item[k]==val for k, val in pattern.items())
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
@@ -30,6 +32,9 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
float
"""
if result is None:
return 0.
expect_metrics = [False] * len(rules.get("expect", []))
unexpect_metric = True
with open(result) as f:
@@ -55,6 +60,9 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float:
float
"""
if result is None:
return 0.
expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])]
unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])]

125
desktop_env/evaluators/metrics/thunderbird.py
View File

@@ -1,10 +1,11 @@
from typing import List, Pattern, Dict, Match
from typing import Union, Any, Optional, Iterable, TypeVar
from typing import List, Pattern, Dict, Match, Tuple
from typing import Union, Any, Optional, Iterable, TypeVar, Callable
import re
import functools
import operator
import json
from .utils import _match_record
import logging
logger = logging.getLogger("desktopenv.metric.thunderbird")
@@ -82,20 +83,128 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any
return float(all(expect_metrics.values()) and unexpect_metric)
_value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\")
#_condition_pattern: Pattern[str] = re.compile(r'(?P<type>AND|OR) \((?P<key>[\w ]+),(?P<rel>[\w ' + '\'' + r']+),(?:"(?P<val2>(?:[^"]|\")+)"|(?P<val1>[^)]+))\)')
_condition_pattern: Pattern[str] = re.compile(r'(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)')
def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
Args:
result (str): path to filter def file
rules (Dict[str, List[Dict[str, str]]]): dict like
{
"expect": [{key: value}]
"unexpect": [{key: value}]
}
Returns:
float
"""
if result is None:
return 0.
# read filter def file
# a filter:
# {
# "name": "Name",
# "enabled": "yes" | "no",
# "type": "17",
# "action": "Move to folder" | ...,
# "actionValue": ...,
# "condition": [...]
# }
filters: List[Dict[str, Union[str, List[str]]]] = []
with open(result) as f:
for l in f:
if l.startswith("name="):
filter_: Dict[str, Union[str, List[str]]] = {}
filter_["name"] = _value_processor(l[6:-2])
elif l.startswith("enabled="):
filter_["enabled"] = _value_processor(l[9:-2])
elif l.startswith("type="):
filter_["type"] = _value_processor(l[6:-2])
elif l.startswith("action="):
filter_["action"] = _value_processor(l[8:-2])
elif l.startswith("actionValue="):
filter_["actionValue"] = _value_processor(l[13:-2])
elif l.startswith("condition="):
condition_str: str = _value_processor(l[11:-2])
logger.debug("FILTER CONDITION: %s", condition_str)
conditions: List[str] =\
_condition_pattern.findall(condition_str)
logger.debug("FILTER CONDITIONS: %s", repr(conditions))
filter_["condition"] = conditions
logger.debug("FILTER %s", repr(filter_))
filters.append(filter_)
expect_metrics = [False] * len(rules.get("expect", []))
unexpect_metric = True
for flt in filters:
for i, r in enumerate(rules.get("expect", [])):
expect_metrics[i] = expect_metrics[i] or _match_record(r, flt)
unexpect_metric = unexpect_metric and not any(_match_record(r, flt) for r in rules.get("unexpect", []))
return float(all(expect_metrics) and unexpect_metric)
if __name__ == "__main__":
import lxml.etree
from lxml.cssselect import CSSSelector
from lxml.etree import _Element
#xml = "../../任务数据/Thunderbird/vertical-card-view.xml"
xml = "../../任务数据/Thunderbird/vertical-table-view.xml"
at: _Element = lxml.etree.parse(xml)
#xml = "../../任务数据/Thunderbird/vertical-table-view.xml"
#at: _Element = lxml.etree.parse(xml)
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
print(len(elements))
for elm in elements:
print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
#print(len(elements))
#for elm in elements:
#print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))
import datetime
import os
import sys
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str)))
debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str)))
stdout_handler = logging.StreamHandler(sys.stdout)
sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)))
file_handler.setLevel(logging.INFO)
debug_handler.setLevel(logging.DEBUG)
stdout_handler.setLevel(logging.INFO)
sdebug_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
file_handler.setFormatter(formatter)
debug_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
sdebug_handler.setFormatter(formatter)
stdout_handler.addFilter(logging.Filter("desktopenv"))
sdebug_handler.addFilter(logging.Filter("desktopenv"))
logger.addHandler(file_handler)
logger.addHandler(debug_handler)
logger.addHandler(stdout_handler)
logger.addHandler(sdebug_handler)
print( check_thunderbird_filter( "../../任务数据/Thunderbird/msgFilterRules.dat"
, { "expect": [ { "enabled": "yes"
, "action": "Move to folder"
, "actionValue": "mailbox://nobody@Local%20Folders/Promotions"
, "condition": ["AND (subject,contains,discount)"]
}
]
}
)
)

3
desktop_env/evaluators/metrics/utils.py
View File

@@ -125,6 +125,9 @@ def load_charts(xlsx_file: Workbook, **options) -> Dict[str, Any]:
chart_set[series] = info
return chart_set
def _match_record(pattern: Dict[str, Any], item: Dict[str, Any]) -> bool:
return all(k in item and item[k]==val for k, val in pattern.items())
if __name__ == "__main__":
path1 = "../../../../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold_line_scatter.xlsx"
workbook1: Workbook = openpyxl.load_workbook(filename=path1)

77
evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json Normal file
View File

@@ -0,0 +1,77 @@
{
"id": "e1e75309-3ddb-4d09-92ec-de869c928143",
"snapshot": "thunderbird",
"instruction": "Create a local folder called \"Promotions\" and create a filter to auto move the inbox emails whose subject contains “discount” to the new folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/e1e75309-3ddb-4d09-92ec-de869c928143",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "command",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Move to folder",
"actionValue": "mailbox://nobody@Local%20Folders/Promotions",
"condition": ["AND (subject,contains,discount)"]
}
]
}
}
}
}

55
evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup Normal file
View File

@@ -0,0 +1,55 @@
{
"id": "e1e75309-3ddb-4d09-92ec-de869c928143",
"snapshot": "thunderbird",
"instruction": "Create a local folder called \"Promotions\" and create a filter to auto move the inbox emails whose subject contains “discount” to the new folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
}
],
"trajectory": "trajectories/e1e75309-3ddb-4d09-92ec-de869c928143",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "command",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Move to folder",
"actionValue": "mailbox://nobody@Local%20Folders/Promotions",
"condition": ["AND (subject,contains,discount)"]
}
]
}
}
}
}

4
main.py
View File

@@ -44,9 +44,9 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/thunderbird/6766f2b8-8a72-417f-a9e5-56fcaa735837.json.nosetup", "r") as f:
with open("evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup", "r") as f:
example = json.load(f)
example["snapshot"] = "Snapshot 17"
example["snapshot"] = "Snapshot 18"
env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
, action_space="computer_13"