diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 28a53ae..3148d2b 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -9,5 +9,5 @@ from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom from .table import compare_table from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ compare_videos -from .general import check_csv, check_accessibility_tree, check_list +from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index d0795a6..e1e2c95 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -13,6 +13,8 @@ from rapidfuzz import fuzz from .utils import _match_record +import sqlite3 + def exact_match(result, rules) -> float: expect = rules["expected"] print(result, expect) @@ -146,3 +148,8 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float: # def check_existence(result: str, *args) -> float: # return 1. - (result is None) + +def run_sqlite3(result: str, rules: Dict[str, Any]) -> float: + connection = sqlite3.connect(result) + connection.execute(rules["sql"]) + return float(connection.fetchone()[0] or 0) diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json new file mode 100644 index 0000000..4f688d6 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json @@ -0,0 +1,74 @@ +{ + "id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "snapshot": "thunderbird", + "instruction": "Add a star to every email in local Bills folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xzv", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "wmctrl", + "-xFc", + "Mail.thunderbird" + ], + "until": { + "returncode": 1 + } + } + } + ], + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite", + "dest": "global-messages-db.sqlite" + }, + "func": "run_sqlite3", + "expected": { + "type": "rule", + "rules": { + "sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);" + } + } + } +} diff --git a/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup new file mode 100644 index 0000000..7c1e728 --- /dev/null +++ b/evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup @@ -0,0 +1,52 @@ +{ + "id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "snapshot": "thunderbird", + "instruction": "Add a star to every email in local Bills folder", + "source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + } + ], + "trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", + "related_apps": [ + "thunderbird" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "wmctrl", + "-xFc", + "Mail.thunderbird" + ], + "until": { + "returncode": 1 + } + } + } + ], + "result": { + "type": "vm_file", + "path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite", + "dest": "global-messages-db.sqlite" + }, + "func": "run_sqlite3", + "expected": { + "type": "rule", + "rules": { + "sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);" + } + } + } +}