lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix id

Browse Source
This commit is contained in:
Timothyxxx
2025-03-03 18:26:26 +08:00
parent c4d818c5cf
commit 13127de01e
15 changed files with 1157 additions and 1144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
evaluation_examples/examples/thunderbird/08c73485-7c6d-4681-999d-919f5c32dcfa.json
View File

@@ -1,90 +1,90 @@
{
"id": "c9e7eaf2-b1a1-4efc-a982-721972fa9f02",
"snapshot": "thunderbird",
"instruction": "Thunderbird's message filters seem to only fire on Inbox automatically. If you want to filter on subfolders, you'd have to start this filter manually. I am wondering if the filter can be applied automatically. Could you help me apply automatic message filters to subfolders",
"source": "https://superuser.com/questions/544480/how-to-apply-automatic-message-filters-to-subfolders-too?noredirect=1&lq=1",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/c9e7eaf2-b1a1-4efc-a982-721972fa9f02",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.server.default.applyIncomingFilters": {
"method": "eq",
"ref": true
},
"mail.imap.use_status_for_biff": {
"method": "eq",
"ref": false
}
"id": "08c73485-7c6d-4681-999d-919f5c32dcfa",
"snapshot": "thunderbird",
"instruction": "Thunderbird's message filters seem to only fire on Inbox automatically. If you want to filter on subfolders, you'd have to start this filter manually. I am wondering if the filter can be applied automatically. Could you help me apply automatic message filters to subfolders",
"source": "https://superuser.com/questions/544480/how-to-apply-automatic-message-filters-to-subfolders-too?noredirect=1&lq=1",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
"unexpect": {
"mail.server.default.autosync_offline_stores": {
"method": "eq",
"ref": false
}
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.server.default.applyIncomingFilters": {
"method": "eq",
"ref": true
},
"mail.imap.use_status_for_biff": {
"method": "eq",
"ref": false
}
},
"unexpect": {
"mail.server.default.autosync_offline_stores": {
"method": "eq",
"ref": false
}
}
}
}
}
}
}
}
}

150
evaluation_examples/examples/thunderbird/10a730d5-d414-4b40-b479-684bed1ae522.json
View File

@@ -1,80 +1,80 @@
{
"id": "94760984-3ff5-41ee-8347-cf1af709fea0",
"snapshot": "thunderbird",
"instruction": "Considering I work late into the ight and use Thunderbird frequently, I find that a full dark mode would be easier on my eyes during those hours. Can you help me enable a complete dark mode in Thunderbird?",
"source": "https://superuser.com/questions/1757333/how-can-i-view-thunderbird-in-full-dark-mode",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/94760984-3ff5-41ee-8347-cf1af709fea0",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
"id": "10a730d5-d414-4b40-b479-684bed1ae522",
"snapshot": "thunderbird",
"instruction": "Considering I work late into the ight and use Thunderbird frequently, I find that a full dark mode would be easier on my eyes during those hours. Can you help me enable a complete dark mode in Thunderbird?",
"source": "https://superuser.com/questions/1757333/how-can-i-view-thunderbird-in-full-dark-mode",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"extensions.activeThemeID": {
"method": "re",
"ref": "dark"
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"extensions.activeThemeID": {
"method": "re",
"ref": "dark"
}
}
}
}
}
}
}
}
}

189
evaluation_examples/examples/thunderbird/15c3b339-88f7-4a86-ab16-e71c58dcb01e.json
View File

@@ -1,100 +1,99 @@
{
"id": "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"snapshot": "thunderbird",
"instruction": "Help me access my outlook account with address \"anonym-x2024@outlook.com\" and password 'Wlv(z._6y|a,rrjfQuQhIi\\$;' (without ')",
"source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058",
"path": "/home/user/thunderbird-profile-blank.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile-blank.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/firefox_decrypt.py"
"id": "15c3b339-88f7-4a86-ab16-e71c58dcb01e",
"snapshot": "thunderbird",
"instruction": "Help me access my outlook account with address \"anonym-x2024@outlook.com\" and password 'Wlv(z._6y|a,rrjfQuQhIi\\$;' (without ')",
"source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058",
"path": "/home/user/Desktop/thunderbird-profile-blank.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"-f",
"/home/user/Desktop/thunderbird-profile-blank.tar.gz",
"-C",
"/home/user/Desktop/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/user/firefox_decrypt.py",
"/home/user/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "Wlv(z._6y|a,rrjfQuQhIi\\$;"
},
{
"url": "smtp://smtp.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "Wlv(z._6y|a,rrjfQuQhIi\\$;"
}
]
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/Desktop/firefox_decrypt.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/user/Desktop/firefox_decrypt.py",
"/home/user/Desktop/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "Wlv(z._6y|a,rrjfQuQhIi\\$;"
},
{
"url": "smtp://smtp.office365.com",
"user": "anonym-x2024@outlook.com",
"password": "Wlv(z._6y|a,rrjfQuQhIi\\$;"
}
]
}
}
}
}
}
}

154
evaluation_examples/examples/thunderbird/3f28fe4f-5d9d-4994-a456-efd78cfae1a3.json
View File

@@ -1,80 +1,80 @@
{
"id": "6766f2b8-8a72-417f-a9e5-56fcaa735837",
"snapshot": "thunderbird",
"instruction": "Set up a signature using my name and affiliation. My name is Anonym and my affiliation is XYZ Lab.",
"source": "https://www.adsigner.com/user-manual/signatures/setup-email-client-thunderbird/#:~:text=is%20probably%20hidden.-,Right%20click%20on%20the%20empty%20space%20at%20the%20top%20of,signature%20from%20a%20file%20instead.",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/6766f2b8-8a72-417f-a9e5-56fcaa735837",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
"id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3",
"snapshot": "thunderbird",
"instruction": "Set up a signature using my name and affiliation. My name is Anonym and my affiliation is XYZ Lab.",
"source": "https://www.adsigner.com/user-manual/signatures/setup-email-client-thunderbird/#:~:text=is%20probably%20hidden.-,Right%20click%20on%20the%20empty%20space%20at%20the%20top%20of,signature%20from%20a%20file%20instead.",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.identity.id1.htmlSigText": {
"method": "re.S",
"ref": "Anonym.+XYZ Lab"
}
}
}
},
"func": "check_thunderbird_prefs"
}
}
"trajectory": "trajectories/6766f2b8-8a72-417f-a9e5-56fcaa735837",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.identity.id1.htmlSigText": {
"method": "re.S",
"ref": "Anonym.+XYZ Lab"
}
}
}
},
"func": "check_thunderbird_prefs"
}
}

168
evaluation_examples/examples/thunderbird/3f49d2cc-f400-4e7d-90cc-9b18e401cc31.json
View File

@@ -1,85 +1,85 @@
{
"id": "480bcfea-d68f-4aaa-a0a9-2589ef319381",
"snapshot": "thunderbird",
"instruction": "I've got a bunch of email accounts in Thunderbird, and it's a hassle to check them one by one. Can you show me how to set up a unified inbox so I can see all my emails in one place?",
"source": "https://www.reddit.com/r/Thunderbird/comments/182dg5p/unified_inbox_howto/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/480bcfea-d68f-4aaa-a0a9-2589ef319381",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/xulstore.json",
"dest": "xulstore.json"
},
"func": "check_json",
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"key": [
"chrome://messenger/content/messenger.xhtml",
"folderTree",
"mode"
],
"method": "re",
"ref": "\\bsmart\\b"
}
]
}
}
}
}
"id": "3f49d2cc-f400-4e7d-90cc-9b18e401cc31",
"snapshot": "thunderbird",
"instruction": "I've got a bunch of email accounts in Thunderbird, and it's a hassle to check them one by one. Can you show me how to set up a unified inbox so I can see all my emails in one place?",
"source": "https://www.reddit.com/r/Thunderbird/comments/182dg5p/unified_inbox_howto/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/xulstore.json",
"dest": "xulstore.json"
},
"func": "check_json",
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"key": [
"chrome://messenger/content/messenger.xhtml",
"folderTree",
"mode"
],
"method": "re",
"ref": "\\bsmart\\b"
}
]
}
}
}
}

156
evaluation_examples/examples/thunderbird/5203d847-2572-4150-912a-03f062254390.json
View File

@@ -1,81 +1,83 @@
{
"id": "e1e75309-3ddb-4d09-92ec-de869c928143",
"snapshot": "thunderbird",
"instruction": "Create a local folder called \"Promotions\" and create a filter to auto move the inbox emails whose subject contains “discount” to the new folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/e1e75309-3ddb-4d09-92ec-de869c928143",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Message Filters",
"strict": true
"id": "5203d847-2572-4150-912a-03f062254390",
"snapshot": "thunderbird",
"instruction": "Create a local folder called \"Promotions\" and create a filter to auto move the inbox emails whose subject contains \u201cdiscount\u201d to the new folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Move to folder",
"actionValue": "mailbox://nobody@Local%20Folders/Promotions",
"condition": ["AND (subject,contains,discount)"]
}
]
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Message Filters",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Move to folder",
"actionValue": "mailbox://nobody@Local%20Folders/Promotions",
"condition": [
"AND (subject,contains,discount)"
]
}
]
}
}
}
}
}
}

115
evaluation_examples/examples/thunderbird/7b1e1ff9-bb85-49be-b01d-d6424be18cd0.json
View File

@@ -1,61 +1,60 @@
{
"id": "12086550-11c0-466b-b367-1d9e75b3910e",
"snapshot": "thunderbird",
"instruction": "Could you help me open up the Thunderbird profile manager utility?",
"source": "https://www.quora.com/How-do-I-open-a-Thunderbird-profile-manager-utility",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/12086550-11c0-466b-b367-1d9e75b3910e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"result": {
"type": "accessibility_tree"
},
"expected": {
"type": "rule",
"rules": [
"id": "7b1e1ff9-bb85-49be-b01d-d6424be18cd0",
"snapshot": "thunderbird",
"instruction": "Could you help me open up the Thunderbird profile manager utility?",
"source": "https://www.quora.com/How-do-I-open-a-Thunderbird-profile-manager-utility",
"config": [
{
"selectors": [
"application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]"
]
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/Desktop/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"-f",
"/home/user/Desktop/thunderbird-profile.tar.gz",
"-C",
"/home/user/Desktop/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
]
},
"func": "check_accessibility_tree"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"result": {
"type": "accessibility_tree"
},
"expected": {
"type": "rule",
"rules": [
{
"selectors": [
"application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]"
]
}
]
},
"func": "check_accessibility_tree"
}
}

156
evaluation_examples/examples/thunderbird/9b7bc335-06b5-4cd3-9119-1a649c478509.json
View File

@@ -1,81 +1,83 @@
{
"id": "35253b65-1c19-4304-8aa4-6884b8218fc0",
"snapshot": "thunderbird",
"instruction": "Set up to forward every email received by anonym-x2024@outlook.com in the future to anonym-x2024@gmail.com. Please don't touch the online account. Just locally in the Thunderbird!",
"source": "https://support.mozilla.org/en-US/questions/1259354",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/35253b65-1c19-4304-8aa4-6884b8218fc0",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Message Filters",
"strict": true
"id": "9b7bc335-06b5-4cd3-9119-1a649c478509",
"snapshot": "thunderbird",
"instruction": "Set up to forward every email received by anonym-x2024@outlook.com in the future to anonym-x2024@gmail.com. Please don't touch the online account. Just locally in the Thunderbird!",
"source": "https://support.mozilla.org/en-US/questions/1259354",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Forward",
"actionValue": "anonym-x2024@gmail.com",
"condition": ["ALL"]
}
]
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Message Filters",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Forward",
"actionValue": "anonym-x2024@gmail.com",
"condition": [
"ALL"
]
}
]
}
}
}
}
}
}

136
evaluation_examples/examples/thunderbird/9bc3cc16-074a-45ac-9bdc-b2a362e1daf3.json
View File

@@ -1,70 +1,74 @@
{
"id": "06fe7178-4491-4589-810f-2e2bc9502122",
"snapshot": "thunderbird",
"instruction": "Could you help me back up all the email files in my inbox to ~/emails.bak? Please save them separately in eml format.",
"source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/06fe7178-4491-4589-810f-2e2bc9502122",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "command",
"parameters": {
"command": ["ls", "-R", "/home/user/emails.bak"],
"stdout": "emails.bak.ls"
"id": "9bc3cc16-074a-45ac-9bdc-b2a362e1daf3",
"snapshot": "thunderbird",
"instruction": "Could you help me back up all the email files in my inbox to ~/emails.bak? Please save them separately in eml format.",
"source": "https://www.quora.com/How-do-I-backup-email-files-in-Mozilla-Thunderbird",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "emails.bak.ls"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"歡迎使用新的 Outlook.com 帳戶.*\\.eml",
"A Test E-mail.*\\.eml"
]
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "command",
"parameters": {
"command": [
"ls",
"-R",
"/home/user/emails.bak"
],
"stdout": "emails.bak.ls"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "emails.bak.ls"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"\u6b61\u8fce\u4f7f\u7528\u65b0\u7684 Outlook.com \u5e33\u6236.*\\.eml",
"A Test E-mail.*\\.eml"
]
}
}
}
}
}
}

140
evaluation_examples/examples/thunderbird/a10b69e1-6034-4a2b-93e1-571d45194f75.json
View File

@@ -1,72 +1,76 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "thunderbird",
"instruction": "Create two local folders in Thunderbird for me: COMPANY and UNIVERSITY.",
"source": "https://support.mozilla.org/bm/questions/1027435",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": ["ls", "-R", "/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders"],
"stdout": "thunder-local-folder.ls"
"id": "a10b69e1-6034-4a2b-93e1-571d45194f75",
"snapshot": "thunderbird",
"instruction": "Create two local folders in Thunderbird for me: COMPANY and UNIVERSITY.",
"source": "https://support.mozilla.org/bm/questions/1027435",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "thunder-local-folder.ls"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"\\bCOMPANY\\.msf\\b",
"\\bCOMPANY/?(?!\\.msf)",
"\\bUNIVERSITY\\.msf\\b",
"\\bUNIVERSITY/?(?!\\.msf)"
]
}
"trajectory": "trajectories/2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"ls",
"-R",
"/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders"
],
"stdout": "thunder-local-folder.ls"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "thunder-local-folder.ls"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"\\bCOMPANY\\.msf\\b",
"\\bCOMPANY/?(?!\\.msf)",
"\\bUNIVERSITY\\.msf\\b",
"\\bUNIVERSITY/?(?!\\.msf)"
]
}
}
}
}
}
}

92
evaluation_examples/examples/thunderbird/a1af9f1c-50d5-4bc3-a51e-4d9b425ff638.json
View File

@@ -1,48 +1,48 @@
{
"id": "99146c54-4f37-4ab8-9327-5f3291665e1e",
"snapshot": "thunderbird",
"instruction": "Due to certain security considerations and the nature of my work, I prefer not to configure an incoming email service in Thunderbird. However, I still need to send emails. Can you help me set up Thunderbird to send emails from anonym-x2024@outlook.com without configuring its incoming email service?",
"source": "https://superuser.com/questions/1764409/how-to-send-email-with-thunderbird-without-configuring-an-incoming-email-service",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058",
"path": "/home/user/thunderbird-profile-blank.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile-blank.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
"id": "a1af9f1c-50d5-4bc3-a51e-4d9b425ff638",
"snapshot": "thunderbird",
"instruction": "Due to certain security considerations and the nature of my work, I prefer not to configure an incoming email service in Thunderbird. However, I still need to send emails. Can you help me set up Thunderbird to send emails from anonym-x2024@outlook.com without configuring its incoming email service?",
"source": "https://superuser.com/questions/1764409/how-to-send-email-with-thunderbird-without-configuring-an-incoming-email-service",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1hSVXjep_RBaN2VN039sKCkFM5NB32wn9&export=download&authuser=0&confirm=t&uuid=90be5a47-f360-4460-8706-628ae555e52b&at=APZUnTV7JqHDhcQJjWKJL_2jO1vj:1705411196058",
"path": "/home/user/thunderbird-profile-blank.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile-blank.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/99146c54-4f37-4ab8-9327-5f3291665e1e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"func": "infeasible"
}
],
"trajectory": "trajectories/99146c54-4f37-4ab8-9327-5f3291665e1e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"func": "infeasible"
}
}
}

198
evaluation_examples/examples/thunderbird/d38192b0-17dc-4e1d-99c3-786d0117de77.json
View File

@@ -1,98 +1,102 @@
{
"id": "d088f539-cab4-4f9a-ac92-9999fc3a656e",
"snapshot": "thunderbird",
"instruction": "Attach the my AWS bill to the email. The bill is stored at ~/aws-bill.pdf. Don't close it or send it. I haven't finish all the contents.",
"source": "https://support.mozilla.org/en-US/kb/how-use-attachments",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1JDUxe8Cs0D9La9ECoYbHJAIQ95DZfkDT&export=download&authuser=0&confirm=t&uuid=1e0f94d8-4fb1-4f9b-878a-516cf4a7ebc4&at=APZUnTXHBZKTuW_vQUzi-1XKuAYg:1705242554801",
"path": "/home/user/aws-bill.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
},
{
"url": "https://drive.usercontent.google.com/download?id=17LutYTkEXhM__sxvrdldar7p-eQba4Pm&export=download&authuser=0&confirm=t&uuid=34730e1c-0fac-401a-ab91-102a47f02224&at=APZUnTX4eFTnG9vnYlQEJX6N0jqP:1705227498314",
"path": "/home/user/.aws-bill-mail-body.html"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',to=assistant@outlook.com,subject='New-month AWS Bill',body='$(cat /home/user/.aws-bill-mail-body.html)'\"",
"shell": true
}
}
],
"trajectory": "trajectories/d088f539-cab4-4f9a-ac92-9999fc3a656e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1abb7PPPRGGy3elpq2vTXe2aKTMOrbm7c&export=download&authuser=0&confirm=t&uuid=652a3d4c-278e-499c-bb26-58d0f70aa317&at=APZUnTXs_ZTZzOF3QrBa_rMjMr08:1705287009269",
"path": "/home/user/show-thunderbird-attachments.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": ["pip", "install", "cssselect"]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"/home/user/show-thunderbird-attachments.py",
"New-month AWS Bill",
"aws-bill.pdf"
],
"stdout": "check-attachments.output"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "check-attachments.output"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"Attachment added!"
]
}
}
}
}
"id": "d38192b0-17dc-4e1d-99c3-786d0117de77",
"snapshot": "thunderbird",
"instruction": "Attach the my AWS bill to the email. The bill is stored at ~/aws-bill.pdf. Don't close it or send it. I haven't finish all the contents.",
"source": "https://support.mozilla.org/en-US/kb/how-use-attachments",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1JDUxe8Cs0D9La9ECoYbHJAIQ95DZfkDT&export=download&authuser=0&confirm=t&uuid=1e0f94d8-4fb1-4f9b-878a-516cf4a7ebc4&at=APZUnTXHBZKTuW_vQUzi-1XKuAYg:1705242554801",
"path": "/home/user/aws-bill.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
},
{
"url": "https://drive.usercontent.google.com/download?id=17LutYTkEXhM__sxvrdldar7p-eQba4Pm&export=download&authuser=0&confirm=t&uuid=34730e1c-0fac-401a-ab91-102a47f02224&at=APZUnTX4eFTnG9vnYlQEJX6N0jqP:1705227498314",
"path": "/home/user/.aws-bill-mail-body.html"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',to=assistant@outlook.com,subject='New-month AWS Bill',body='$(cat /home/user/.aws-bill-mail-body.html)'\"",
"shell": true
}
}
],
"trajectory": "trajectories/d088f539-cab4-4f9a-ac92-9999fc3a656e",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1abb7PPPRGGy3elpq2vTXe2aKTMOrbm7c&export=download&authuser=0&confirm=t&uuid=652a3d4c-278e-499c-bb26-58d0f70aa317&at=APZUnTXs_ZTZzOF3QrBa_rMjMr08:1705287009269",
"path": "/home/user/show-thunderbird-attachments.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"pip",
"install",
"cssselect"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"/home/user/show-thunderbird-attachments.py",
"New-month AWS Bill",
"aws-bill.pdf"
],
"stdout": "check-attachments.output"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "check-attachments.output"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"Attachment added!"
]
}
}
}
}

148
evaluation_examples/examples/thunderbird/dd84e895-72fd-4023-a336-97689ded257c.json
View File

@@ -1,75 +1,75 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}
"id": "dd84e895-72fd-4023-a336-97689ded257c",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}

177
evaluation_examples/examples/thunderbird/dfac9ee8-9bc4-4cdc-b465-4a4bfcd2f397.json
View File

@@ -1,94 +1,93 @@
{
"id": "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"snapshot": "thunderbird",
"instruction": "Help me to remove the account \"anonym-x2024@outlook.com\"",
"source": "https://www.wikihow.com/Remove-an-Email-Account-from-Thunderbird",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/firefox_decrypt.py"
"id": "dfac9ee8-9bc4-4cdc-b465-4a4bfcd2f397",
"snapshot": "thunderbird",
"instruction": "Help me to remove the account \"anonym-x2024@outlook.com\"",
"source": "https://www.wikihow.com/Remove-an-Email-Account-from-Thunderbird",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/Desktop/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"-f",
"/home/user/Desktop/thunderbird-profile.tar.gz",
"-C",
"/home/user/Desktop/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/user/firefox_decrypt.py",
"/home/user/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"type": "rule",
"rules": {
"unexpect": [
{
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com"
}
]
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://raw.githubusercontent.com/unode/firefox_decrypt/3f1a6dce63056c1f64d845ff077fc1e653e757c6/firefox_decrypt.py",
"path": "/home/user/Desktop/firefox_decrypt.py"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python3",
"/home/user/Desktop/firefox_decrypt.py",
"/home/user/Desktop/.thunderbird",
"-n",
"-c",
"2",
"-f",
"csv",
"-d",
","
],
"stdout": "thunderbird-accounts.csv"
}
}
],
"func": "check_csv",
"result": {
"type": "cache_file",
"path": "thunderbird-accounts.csv"
},
"expected": {
"type": "rule",
"rules": {
"unexpect": [
{
"url": "imap://outlook.office365.com",
"user": "anonym-x2024@outlook.com"
}
]
}
}
}
}
}
}

150
evaluation_examples/examples/thunderbird/f201fbc3-44e6-46fc-bcaa-432f9815454c.json
View File

@@ -1,80 +1,80 @@
{
"id": "030eeff7-b492-4218-b312-701ec99ee0cc",
"snapshot": "thunderbird",
"instruction": "When I reply to an email, it quotes the original message but offsets it with an indentation and \">\" character. I would like to quote the original message with no indentation, and no special character. Could you help me remove the indentation and \">\" for me?",
"source": "https://superuser.com/questions/1781004/how-do-i-remove-the-indentation-and-character-in-quoted-text-of-a-reply-mess",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/030eeff7-b492-4218-b312-701ec99ee0cc",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
"id": "f201fbc3-44e6-46fc-bcaa-432f9815454c",
"snapshot": "thunderbird",
"instruction": "When I reply to an email, it quotes the original message but offsets it with an indentation and \">\" character. I would like to quote the original message with no indentation, and no special character. Could you help me remove the indentation and \">\" for me?",
"source": "https://superuser.com/questions/1781004/how-do-i-remove-the-indentation-and-character-in-quoted-text-of-a-reply-mess",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.identity.id1.auto_quote": {
"method": "eq",
"ref": false
}
"trajectory": "trajectories/",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "close_window",
"parameters": {
"window_name": "Mail.thunderbird",
"strict": true,
"by_class": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_thunderbird_prefs",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/prefs.js",
"dest": "thunder-prefs.js"
},
"expected": {
"type": "rule",
"rules": {
"expect": {
"mail.identity.id1.auto_quote": {
"method": "eq",
"ref": false
}
}
}
}
}
}
}
}
}