From 5160619783573cdcfbccbe8b9e7b9c4c255bfba4 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 12 Jan 2024 12:19:23 +0800 Subject: [PATCH 1/3] ver Jan12th quickly fixed two thunderbird examples --- desktop_env/evaluators/metrics/thunderbird.py | 2 +- .../2ad9387a-65d8-4e33-ad5b-7580065a27ca.json | 72 +++++++++++++++++ .../35253b65-1c19-4304-8aa4-6884b8218fc0.json | 77 +++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json create mode 100644 evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json diff --git a/desktop_env/evaluators/metrics/thunderbird.py b/desktop_env/evaluators/metrics/thunderbird.py index b202d94..ce7d00d 100644 --- a/desktop_env/evaluators/metrics/thunderbird.py +++ b/desktop_env/evaluators/metrics/thunderbird.py @@ -85,7 +85,7 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any _value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\") #_condition_pattern: Pattern[str] = re.compile(r'(?PAND|OR) \((?P[\w ]+),(?P[\w ' + '\'' + r']+),(?:"(?P(?:[^"]|\")+)"|(?P[^)]+))\)') -_condition_pattern: Pattern[str] = re.compile(r'(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)') +_condition_pattern: Pattern[str] = re.compile(r'\b(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)|\bALL\b') def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ Args: diff --git a/evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json b/evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json new file mode 100644 index 0000000..6b89f31 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json @@ -0,0 +1,72 @@ +{ + "id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca", + "snapshot": "thunderbird", + "instruction": "Create two local folders for me: COMPANY and UNIVERSITY.", + "source": "https://support.mozilla.org/bm/questions/1027435", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/2ad9387a-65d8-4e33-ad5b-7580065a27ca", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": ["ls", "-R", "/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders"], + "stdout": "thunder-local-folder.ls" + } + } + ], + "func": "check_list", + "result": { + "type": "cache_file", + "path": "thunder-local-folder.ls" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + "\\bCOMPANY\\.msf\\b", + "\\bCOMPANY/?(?!\\.msf)", + "\\bUNIVERSITY\\.msf\\b", + "\\bUNIVERSITY/?(?!\\.msf)" + ] + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json b/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json new file mode 100644 index 0000000..a1c3154 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json @@ -0,0 +1,77 @@ +{ + "id": "35253b65-1c19-4304-8aa4-6884b8218fc0", + "snapshot": "thunderbird", + "instruction": "Set up to forward every email received by anonym-x2024@outlook.com in the future to anonym-x2024@gmail.com. Please don't touch the online account. Just locally in the Thunderbird!", + "source": "https://support.mozilla.org/en-US/questions/1259354", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/35253b65-1c19-4304-8aa4-6884b8218fc0", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "command", + "parameters": { + "command": ["wmctrl", "-Fc", "Message Filters"], + "until": { + "returncode": 1 + } + } + } + ], + "func": "check_thunderbird_filter", + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat", + "dest": "msgFilterRules.dat" + }, + "expected": { + "type": "rule", + "rules": { + "expect": [ + { + "enabled": "yes", + "action": "Forward", + "actionValue": "anonym-x2024@gmail.com", + "condition": ["ALL"] + } + ] + } + } + } +} From e08df5712912379f41d4f51f9eb996addadbdc49 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 12 Jan 2024 23:07:00 +0800 Subject: [PATCH 2/3] ver Jan12thv2 sqlite3 metric --- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/general.py | 7 ++ .../3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json | 74 +++++++++++++++++++ ...7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup | 52 +++++++++++++ 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json create mode 100644 evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 28a53ae..3148d2b 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -9,5 +9,5 @@ from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom from .table import compare_table from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ compare_videos -from .general import check_csv, check_accessibility_tree, check_list +from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index d0795a6..e1e2c95 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -13,6 +13,8 @@ from rapidfuzz import fuzz from .utils import _match_record +import sqlite3 + def exact_match(result, rules) -> float: expect = rules["expected"] print(result, expect) @@ -146,3 +148,8 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: # def check_existence(result: str, *args) -> float: # return 1. - (result is None) + +def run_sqlite3(result: str, rules: Dict[str, Any]) -> float: + connection = sqlite3.connect(result) + connection.execute(rules["sql"]) + return float(connection.fetchone()[0] or 0) diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json new file mode 100644 index 0000000..4f688d6 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json @@ -0,0 +1,74 @@ +{ + "id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "snapshot": "thunderbird", + "instruction": "Add a star to every email in local Bills folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "wmctrl", + "-xFc", + "Mail.thunderbird" + ], + "until": { + "returncode": 1 + } + } + } + ], + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite", + "dest": "global-messages-db.sqlite" + }, + "func": "run_sqlite3", + "expected": { + "type": "rule", + "rules": { + "sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);" + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup new file mode 100644 index 0000000..7c1e728 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup @@ -0,0 +1,52 @@ +{ + "id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "snapshot": "thunderbird", + "instruction": "Add a star to every email in local Bills folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + } + ], + "trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "wmctrl", + "-xFc", + "Mail.thunderbird" + ], + "until": { + "returncode": 1 + } + } + } + ], + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite", + "dest": "global-messages-db.sqlite" + }, + "func": "run_sqlite3", + "expected": { + "type": "rule", + "rules": { + "sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);" + } + } + } +} From d4192d3d9c24032c2d673732ce47ac28ff994761 Mon Sep 17 00:00:00 2001 From: David Chang Date: Sat, 13 Jan 2024 00:06:11 +0800 Subject: [PATCH 3/3] ver Jan12thv3 debugged --- desktop_env/evaluators/metrics/general.py | 6 +++--- main.py | 4 ++-- requirements.txt | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index e1e2c95..5b5492e 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -150,6 +150,6 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: # return 1. - (result is None) def run_sqlite3(result: str, rules: Dict[str, Any]) -> float: - connection = sqlite3.connect(result) - connection.execute(rules["sql"]) - return float(connection.fetchone()[0] or 0) + connection: sqlite3.Connection = sqlite3.connect(result) + cursor: sqlite3.Cursor = connection.execute(rules["sql"]) + return float(cursor.fetchone()[0] or 0) diff --git a/main.py b/main.py index 5da8091..6f5650a 100644 --- a/main.py +++ b/main.py @@ -44,9 +44,9 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup", "r") as f: + with open("evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 18" + example["snapshot"] = "Snapshot 20" env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" , action_space="computer_13" diff --git a/requirements.txt b/requirements.txt index 558098b..588ec1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ pyacoustid opencv-python ImageHash scikit-image +librosa