lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into zdy

Browse Source
This commit is contained in:
David Chang
2024-01-13 17:15:32 +08:00
parent d4192d3d9c 136b52c876
commit 005b054a0b
61 changed files with 4616 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
evaluation_examples/examples/chrome/06fe7178-4491-4589-810f-2e2bc9502122.json
View File

@@ -3,16 +3,64 @@
"snapshot": "chrome",
"instruction": "Can you make my computer bring back the last tab I shut down?",
"source": "https://www.wikihow.com/Switch-Tabs-in-Chrome",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.lonelyplanet.com",
"https://www.airbnb.com",
"https://www.tripadvisor.com"
]
}
},
{
"type": "chrome_close_tabs",
"parameters": {
"urls_to_close": [
"https://www.tripadvisor.com"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "is_expected_tabs",
"result": {
"type": "open_tabs_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"urls": [
"https://www.lonelyplanet.com",
"https://www.airbnb.com",
"https://www.tripadvisor.com"
]
}
}
}
}

43
evaluation_examples/examples/chrome/2ad9387a-65d8-4e33-ad5b-7580065a27ca.json Normal file
View File

@@ -0,0 +1,43 @@
{
"id": "2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"snapshot": "chrome",
"instruction": "Can you make a new folder for me on that bookmarks bar in my internet browser? Let's call it 'Favorites.'",
"source": "https://www.youtube.com/watch?v=IN-Eq_UripQ",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_bookmarks",
"result": {
"type": "bookmarks"
},
"expected": {
"type": "rule",
"rules": {
}
}
}
}

18
evaluation_examples/examples/chrome/35253b65-1c19-4304-8aa4-6884b8218fc0.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "35253b65-1c19-4304-8aa4-6884b8218fc0",
"snapshot": "chrome",
"instruction": "Hey, I need a quick way back to this site. Could you whip up a shortcut on my desktop for me?",
"source": "https://www.laptopmag.com/articles/how-to-create-desktop-shortcuts-for-web-pages-using-chrome",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"snapshot": "chrome",
"instruction": "Computer, could you set up Chrome so it automatically fills in forms with my address, say, 123 Main Street, Anytown, USA, for me?",
"source": "https://in5stepstutorials.com/google-chrome/add-change-delete-autofill-address.php",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "6766f2b8-8a72-417f-a9e5-56fcaa735837",
"snapshot": "chrome",
"instruction": "Could you help me install the unpacked extension at /to/path in Chrome?",
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/7a5a7856-f1b6-42a4-ade9-1ca81ca0f263.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "7a5a7856-f1b6-42a4-ade9-1ca81ca0f263",
"snapshot": "chrome",
"instruction": "Can you save this webpage I'm looking at to my bookmarks so I can come back to it later?",
"source": "https://www.youtube.com/watch?v=ZaZ8GcTxjXA",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/d088f539-cab4-4f9a-ac92-9999fc3a656e.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "d088f539-cab4-4f9a-ac92-9999fc3a656e",
"snapshot": "chrome",
"instruction": "I've got a bunch of fun stuff open right now. Can you group all these tabs at the top of Chrome and name it 'Leisure' for me? It'll help me get back to them quickly when I want to relax.",
"source": "https://medium.com/@inkverseuk2/useful-tips-and-tricks-for-the-google-chrome-browser-ac7d0d24b3cc",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

18
evaluation_examples/examples/chrome/e1e75309-3ddb-4d09-92ec-de869c928143.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "e1e75309-3ddb-4d09-92ec-de869c928143",
"snapshot": "chrome",
"instruction": "Computer, can you turn the webpage I'm looking at into a PDF file and put it on my main screen, you know, the Desktop?",
"source": "https://in5stepstutorials.com/google-chrome/save-web-page-as-pdf-in-chrome.php",
"config": [],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"result": {
},
"expected": {
}
}
}

42
evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json Normal file
View File

@@ -0,0 +1,42 @@
{
"id": "554785e9-4523-4e7a-b8e1-8016f565f56a",
"snapshot": "gimp",
"instruction": "Could you help me increase the saturation of my photo to make it more colorful?",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-3cc3qqjlJ_3d9IpwsuvSuYpyJ2LxrIA&export=download&authuser=0&confirm=t&uuid=6b08e902-ed25-47d1-97d9-7794f9508288&at=APZUnTW74TY2hHtBbWDa0N7-cE58:1705060487948",
"path": "Desktop/woman_sitting_by_the_tree2.png"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/woman_sitting_by_the_tree2.png"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "increase_saturation",
"expected": {
"type": "vm_file",
"path": "Desktop/woman_sitting_by_the_tree2.png",
"dest": "woman_sitting_by_the_tree2.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/edited_colorful.png",
"dest": "edited_colorful.png"
}
}
}

34
evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4 Normal file
View File

@@ -0,0 +1,34 @@
{
"id": "77b8ab4d-994f-43ac-8930-8ca087d7c4b4",
"snapshot": "gimp",
"instruction": "Could you help me keep my photo on the desktop and rename it export.jpg?",
"source": "https://superuser.com/questions/1636113/how-to-get-gimp-to-recognize-images-or-pictures-folder-as-the-default-folder-for",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=13ECFsPxznuoCANto21ijj9OzP0APukIH&export=download&authuser=0&confirm=t&uuid=d8f2dd03-8992-4646-be62-3a3cf89583f2&at=APZUnTVsR0xmbXvpFIpXLzCcLrMa:1705062951627",
"path": "Desktop/The_Lost_River_Of_Dreams.jpg"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/The_Lost_River_Of_Dreams.jpg"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "check_file_exists",
"file_name": "export.png",
"directory": "/home/user/Desktop/"
}
}

42
evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json Normal file
View File

@@ -0,0 +1,42 @@
{
"id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568",
"snapshot": "gimp",
"instruction": "Make my picture less bright, please.",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1SIvX9Wimyw6i2UvnoLTNDHIObvDLAsIM&export=download&authuser=0&confirm=t&uuid=a48447ab-13a2-421f-9662-6ffff8f6f6d5&at=APZUnTVRxofs822XxgEv33WwYCkb:1705046264363",
"path": "Desktop/woman_sitting_by_the_tree.png"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/woman_sitting_by_the_tree.png"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "decrease_brightness",
"expected": {
"type": "vm_file",
"path": "Desktop/woman_sitting_by_the_tree.png",
"dest": "woman_sitting_by_the_tree.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/edited_darker.png",
"dest": "edited_darker.png"
}
}
}

42
evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce Normal file
View File

@@ -0,0 +1,42 @@
{
"id": "f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce",
"snapshot": "gimp",
"instruction": "Help me select the yellow triangle and move it to the center of my photo.",
"source": "https://superuser.com/questions/612338/how-do-i-select-and-move-an-object-in-gimp",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1IMFTLy3lLSP8ee5b-940JjaoOd5-oGwn&export=download&authuser=0&confirm=t&uuid=0e06dffa-b602-42c6-8841-cf6eb3524e2b&at=APZUnTVpN4xHqAIAYmHZgk-UXvY4:1705081430356",
"path": "Desktop/Triangle_On_The_Side.png"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/Triangle_On_The_Side.png"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "compare_triangle_positions",
"expected": {
"type": "vm_file",
"path": "Desktop/Triangle_On_The_Side.png",
"dest": "Triangle_On_The_Side.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/Triangle_In_The_Middle.png",
"dest": "Triangle_In_The_Middle.png"
}
}
}