lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-01-13 17:17:51 +08:00
parent 136b52c876 005b054a0b
commit 2f6974a59a
9 changed files with 287 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/evaluators/metrics/__init__.py
View File

@@ -10,8 +10,7 @@ from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
from .table import compare_table
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
compare_videos
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .general import check_csv, check_accessibility_tree, check_list
from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter

7
desktop_env/evaluators/metrics/general.py
View File

@@ -13,6 +13,8 @@ from rapidfuzz import fuzz
from .utils import _match_record
import sqlite3
def exact_match(result, rules) -> float:
expect = rules["expected"]
print(result, expect)
@@ -146,3 +148,8 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
# def check_existence(result: str, *args) -> float:
# return 1. - (result is None)
def run_sqlite3(result: str, rules: Dict[str, Any]) -> float:
connection: sqlite3.Connection = sqlite3.connect(result)
cursor: sqlite3.Cursor = connection.execute(rules["sql"])
return float(cursor.fetchone()[0] or 0)

2
desktop_env/evaluators/metrics/thunderbird.py
View File

@@ -85,7 +85,7 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any
_value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\")
#_condition_pattern: Pattern[str] = re.compile(r'(?P<type>AND|OR) \((?P<key>[\w ]+),(?P<rel>[\w ' + '\'' + r']+),(?:"(?P<val2>(?:[^"]|\")+)"|(?P<val1>[^)]+))\)')
_condition_pattern: Pattern[str] = re.compile(r'(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)')
_condition_pattern: Pattern[str] = re.compile(r'\b(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)|\bALL\b')
def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""
Args:

72
evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json Normal file
View File

@@ -0,0 +1,72 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "thunderbird",
"instruction": "Create two local folders for me: COMPANY and UNIVERSITY.",
"source": "https://support.mozilla.org/bm/questions/1027435",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": ["ls", "-R", "/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders"],
"stdout": "thunder-local-folder.ls"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "thunder-local-folder.ls"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"\\bCOMPANY\\.msf\\b",
"\\bCOMPANY/?(?!\\.msf)",
"\\bUNIVERSITY\\.msf\\b",
"\\bUNIVERSITY/?(?!\\.msf)"
]
}
}
}
}

77
evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json Normal file
View File

@@ -0,0 +1,77 @@
{
"id": "35253b65-1c19-4304-8aa4-6884b8218fc0",
"snapshot": "thunderbird",
"instruction": "Set up to forward every email received by anonym-x2024@outlook.com in the future to anonym-x2024@gmail.com. Please don't touch the online account. Just locally in the Thunderbird!",
"source": "https://support.mozilla.org/en-US/questions/1259354",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/35253b65-1c19-4304-8aa4-6884b8218fc0",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "command",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Forward",
"actionValue": "anonym-x2024@gmail.com",
"condition": ["ALL"]
}
]
}
}
}
}

74
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json Normal file
View File

@@ -0,0 +1,74 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}

52
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup Normal file
View File

@@ -0,0 +1,52 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}

4
main.py
View File

@@ -44,9 +44,9 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/thunderbird/e1e75309-3ddb-4d09-92ec-de869c928143.json.nosetup", "r") as f:
with open("evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup", "r") as f:
example = json.load(f)
example["snapshot"] = "Snapshot 18"
example["snapshot"] = "Snapshot 20"
env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
, action_space="computer_13"

1
requirements.txt
View File

@@ -28,3 +28,4 @@ pyacoustid
opencv-python
ImageHash
scikit-image
librosa