lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:ztjhz/DesktopEnv

Browse Source
This commit is contained in:
Siheng Zhao
2024-01-13 18:14:31 +08:00
parent 105fd35683 2f6974a59a
commit f274193265
31 changed files with 1025 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
View File

@@ -3,16 +3,64 @@
"snapshot": "chrome",
"instruction": "Can you make my computer bring back the last tab I shut down?",
"source": "https://www.wikihow.com/Switch-Tabs-in-Chrome",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.lonelyplanet.com",
"https://www.airbnb.com",
"https://www.tripadvisor.com"
]
}
},
{
"type": "chrome_close_tabs",
"parameters": {
"urls_to_close": [
"https://www.tripadvisor.com"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "is_expected_tabs",
"result": {
"type": "open_tabs_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"urls": [
"https://www.lonelyplanet.com",
"https://www.airbnb.com",
"https://www.tripadvisor.com"
]
}
}
}
}

43
evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json Normal file
View File

@@ -0,0 +1,43 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "chrome",
"instruction": "Can you make a new folder for me on that bookmarks bar in my internet browser? Let's call it 'Favorites.'",
"source": "https://www.youtube.com/watch?v=IN-Eq_UripQ",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_bookmarks",
"result": {
"type": "bookmarks"
},
"expected": {
"type": "rule",
"rules": {
}
}
}
}

18
evaluation_examples/examples/chrome/35253b65-1c19-4304-8aa4-6884b8218fc0.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "35253b65-1c19-4304-8aa4-6884b8218fc0",
"snapshot": "chrome",
"instruction": "Hey, I need a quick way back to this site. Could you whip up a shortcut on my desktop for me?",
"source": "https://www.laptopmag.com/articles/how-to-create-desktop-shortcuts-for-web-pages-using-chrome",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "chrome",
"instruction": "Computer, could you set up Chrome so it automatically fills in forms with my address, say, 123 Main Street, Anytown, USA, for me?",
"source": "https://in5stepstutorials.com/google-chrome/add-change-delete-autofill-address.php",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "6766f2b8-8a72-417f-a9e5-56fcaa735837",
"snapshot": "chrome",
"instruction": "Could you help me install the unpacked extension at /to/path in Chrome?",
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/7a5a7856-f1b6-42a4-ade9-1ca81ca0f263.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "7a5a7856-f1b6-42a4-ade9-1ca81ca0f263",
"snapshot": "chrome",
"instruction": "Can you save this webpage I'm looking at to my bookmarks so I can come back to it later?",
"source": "https://www.youtube.com/watch?v=ZaZ8GcTxjXA",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/d088f539-cab4-4f9a-ac92-9999fc3a656e.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "d088f539-cab4-4f9a-ac92-9999fc3a656e",
"snapshot": "chrome",
"instruction": "I've got a bunch of fun stuff open right now. Can you group all these tabs at the top of Chrome and name it 'Leisure' for me? It'll help me get back to them quickly when I want to relax.",
"source": "https://medium.com/@inkverseuk2/useful-tips-and-tricks-for-the-google-chrome-browser-ac7d0d24b3cc",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/e1e75309-3ddb-4d09-92ec-de869c928143.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "e1e75309-3ddb-4d09-92ec-de869c928143",
"snapshot": "chrome",
"instruction": "Computer, can you turn the webpage I'm looking at into a PDF file and put it on my main screen, you know, the Desktop?",
"source": "https://in5stepstutorials.com/google-chrome/save-web-page-as-pdf-in-chrome.php",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

16
evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json
View File

@@ -9,8 +9,8 @@
"parameters": {
"files": [
{
"url": "",
"path": "Desktop/2.png"
"url": "https://drive.usercontent.google.com/download?id=1-3cc3qqjlJ_3d9IpwsuvSuYpyJ2LxrIA&export=download&authuser=0&confirm=t&uuid=6b08e902-ed25-47d1-97d9-7794f9508288&at=APZUnTW74TY2hHtBbWDa0N7-cE58:1705060487948",
"path": "Desktop/woman_sitting_by_the_tree2.png"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/2.png"
"path": "Desktop/woman_sitting_by_the_tree2.png"
}
}
],
@@ -29,14 +29,14 @@
"evaluator": {
"func": "increase_saturation",
"expected": {
"type": "cloud_file",
"path": "",
"dest": "edited_colorful.png"
"type": "vm_file",
"path": "Desktop/woman_sitting_by_the_tree2.png",
"dest": "woman_sitting_by_the_tree2.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/2.png",
"dest": "2.png"
"path": "Desktop/edited_colorful.png",
"dest": "edited_colorful.png"
}
}
}

34
evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 Normal file
View File

@@ -0,0 +1,34 @@
{
"id": "77b8ab4d-994f-43ac-8930-8ca087d7c4b4",
"snapshot": "gimp",
"instruction": "Could you help me keep my photo on the desktop and rename it export.jpg?",
"source": "https://superuser.com/questions/1636113/how-to-get-gimp-to-recognize-images-or-pictures-folder-as-the-default-folder-for",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=13ECFsPxznuoCANto21ijj9OzP0APukIH&export=download&authuser=0&confirm=t&uuid=d8f2dd03-8992-4646-be62-3a3cf89583f2&at=APZUnTVsR0xmbXvpFIpXLzCcLrMa:1705062951627",
"path": "Desktop/The_Lost_River_Of_Dreams.jpg"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/The_Lost_River_Of_Dreams.jpg"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "check_file_exists",
"file_name": "export.png",
"directory": "/home/user/Desktop/"
}
}

14
evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json
View File

@@ -10,7 +10,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1SIvX9Wimyw6i2UvnoLTNDHIObvDLAsIM&export=download&authuser=0&confirm=t&uuid=a48447ab-13a2-421f-9662-6ffff8f6f6d5&at=APZUnTVRxofs822XxgEv33WwYCkb:1705046264363",
"path": "Desktop/1.png"
"path": "Desktop/woman_sitting_by_the_tree.png"
}
]
}
@@ -18,7 +18,7 @@
{
"type": "open",
"parameters": {
"path": "Desktop/1.png"
"path": "Desktop/woman_sitting_by_the_tree.png"
}
}
],
@@ -29,14 +29,14 @@
"evaluator": {
"func": "decrease_brightness",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1M7T5iPv0kWvmZBbdTcRdG45zIpHMtrBn&export=download&authuser=0&confirm=t&uuid=d95db1f8-27dc-4532-9976-ea99c293f53e&at=APZUnTXfpgNCtXDtlqd55LgmrIAj:1705048270986",
"dest": "edited_darker.png"
"type": "vm_file",
"path": "Desktop/woman_sitting_by_the_tree.png",
"dest": "woman_sitting_by_the_tree.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/1.png",
"dest": "1.png"
"path": "Desktop/edited_darker.png",
"dest": "edited_darker.png"
}
}
}

42
evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce Normal file
View File

@@ -0,0 +1,42 @@
{
"id": "f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce",
"snapshot": "gimp",
"instruction": "Help me select the yellow triangle and move it to the center of my photo.",
"source": "https://superuser.com/questions/612338/how-do-i-select-and-move-an-object-in-gimp",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1IMFTLy3lLSP8ee5b-940JjaoOd5-oGwn&export=download&authuser=0&confirm=t&uuid=0e06dffa-b602-42c6-8841-cf6eb3524e2b&at=APZUnTVpN4xHqAIAYmHZgk-UXvY4:1705081430356",
"path": "Desktop/Triangle_On_The_Side.png"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/Triangle_On_The_Side.png"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "compare_triangle_positions",
"expected": {
"type": "vm_file",
"path": "Desktop/Triangle_On_The_Side.png",
"dest": "Triangle_On_The_Side.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/Triangle_In_The_Middle.png",
"dest": "Triangle_In_The_Middle.png"
}
}
}

72
evaluation_examples/examples/thunderbird/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json Normal file
View File

@@ -0,0 +1,72 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "thunderbird",
"instruction": "Create two local folders for me: COMPANY and UNIVERSITY.",
"source": "https://support.mozilla.org/bm/questions/1027435",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": ["ls", "-R", "/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders"],
"stdout": "thunder-local-folder.ls"
}
}
],
"func": "check_list",
"result": {
"type": "cache_file",
"path": "thunder-local-folder.ls"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
"\\bCOMPANY\\.msf\\b",
"\\bCOMPANY/?(?!\\.msf)",
"\\bUNIVERSITY\\.msf\\b",
"\\bUNIVERSITY/?(?!\\.msf)"
]
}
}
}
}

77
evaluation_examples/examples/thunderbird/35253b65-1c19-4304-8aa4-6884b8218fc0.json Normal file
View File

@@ -0,0 +1,77 @@
{
"id": "35253b65-1c19-4304-8aa4-6884b8218fc0",
"snapshot": "thunderbird",
"instruction": "Set up to forward every email received by anonym-x2024@outlook.com in the future to anonym-x2024@gmail.com. Please don't touch the online account. Just locally in the Thunderbird!",
"source": "https://support.mozilla.org/en-US/questions/1259354",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/35253b65-1c19-4304-8aa4-6884b8218fc0",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "command",
"parameters": {
"command": ["wmctrl", "-Fc", "Message Filters"],
"until": {
"returncode": 1
}
}
}
],
"func": "check_thunderbird_filter",
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/ImapMail/outlook.office365.com/msgFilterRules.dat",
"dest": "msgFilterRules.dat"
},
"expected": {
"type": "rule",
"rules": {
"expect": [
{
"enabled": "yes",
"action": "Forward",
"actionValue": "anonym-x2024@gmail.com",
"condition": ["ALL"]
}
]
}
}
}
}

74
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json Normal file
View File

@@ -0,0 +1,74 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}

52
evaluation_examples/examples/thunderbird/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json.nosetup Normal file
View File

@@ -0,0 +1,52 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "thunderbird",
"instruction": "Add a star to every email in local Bills folder",
"source": "https://support.mozilla.org/en-US/kb/organize-your-messages-using-filters",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=de09bd5e-bef8-499a-b599-c642af190e10&at=APZUnTXqOsQkxl0zMSX6R1Sgp_v3:1704362491712",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
}
],
"trajectory": "trajectories/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"related_apps": [
"thunderbird"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"wmctrl",
"-xFc",
"Mail.thunderbird"
],
"until": {
"returncode": 1
}
}
}
],
"result": {
"type": "vm_file",
"path": "/home/user/.thunderbird/t5q2a5hp.default-release/global-messages-db.sqlite",
"dest": "global-messages-db.sqlite"
},
"func": "run_sqlite3",
"expected": {
"type": "rule",
"rules": {
"sql": "select sum(1) > 0 from messageAttributes where attributeID = 58 and value = 1 and messageID in (select id from messages where folderID = 13);"
}
}
}
}