lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan12thv2

Browse Source 
sqlite3 metric
This commit is contained in:
David Chang
2024-01-12 23:07:00 +08:00
parent 5160619783
commit e08df57129
4 changed files with 134 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -9,5 +9,5 @@ from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
from .table import compare_table from .table import compare_table
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos compare_videos
from .general import check_csv, check_accessibility_tree, check_list from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter

7
desktop_env/evaluators/metrics/general.py
View File

@@ -13,6 +13,8 @@ from rapidfuzz import fuzz
from .utils import _match_record from .utils import _match_record
import sqlite3
def exact_match(result, rules) -> float: def exact_match(result, rules) -> float:
expect = rules["expected"] expect = rules["expected"]
print(result, expect) print(result, expect)
@@ -146,3 +148,8 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
# def check_existence(result: str, *args) -> float: # def check_existence(result: str, *args) -> float:
# return 1. - (result is None) # return 1. - (result is None)
def run_sqlite3(result: str, rules: Dict[str, Any]) -> float:
connection = sqlite3.connect(result)
connection.execute(rules["sql"])
return float(connection.fetchone()[0] or 0)

74
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json Normal file
View File

@@ -0,0 +1,74 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}

52
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup Normal file
View File

@@ -0,0 +1,52 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}