lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into xiaochuanli/addChromeExtensions

Browse Source
This commit is contained in:
Tianbao Xie
2024-02-25 00:45:17 +08:00
committed by GitHub
parent 3244098664 792d8844c7
commit 0a6b5b3f57
152 changed files with 7174 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "0d8b7de3-e8de-4d86-b9fd-dd2dce58a217",
"snapshot": "chrome",
"instruction": "Browse the natural products database.",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://drugs.com"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.drugs.com/npc/"
}
}
}
}

71
evaluation_examples/examples/chrome/121ba48f-9e17-48ce-9bc6-a4fb17a7ebba.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "121ba48f-9e17-48ce-9bc6-a4fb17a7ebba",
"snapshot": "chrome",
"instruction": "Find Dota 2 game and add all DLC to cart.",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.dota2.com/home",
"https://store.steampowered.com"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_added_to_steam_cart",
"result": {
"type": "page_info",
"url": "https://store.steampowered.com/cart/"
},
"expected": {
"type": "rule",
"rules": {
"items": [
"The Dota 2 Official Soundtrack"
]
}
}
}
}

4
evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json
View File

@@ -7,14 +7,14 @@
{
"type": "execute",
"parameters": {
"command": "echo lixiaochuan20 | sudo -S apt update -y && echo lixiaochuan20 | sudo -S apt install jq -y",
"command": "echo password | sudo -S apt update -y && echo password | sudo -S apt install jq -y",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "cd /home/user/snap/chromium/common/chromium/Default && jq '. + {\"session\":{\"restore_on_startup\":4, \"startup_urls\":[\"http://funbrain.com/\"]}}' Preferences > temp && mv temp Preferences",
"command": "cd /home/user/.config/google-chrome/Default && jq '. + {\"session\":{\"restore_on_startup\":4, \"startup_urls\":[\"http://funbrain.com/\"]}}' Preferences > temp && mv temp Preferences",
"shell": true
}
},

2
evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json
View File

@@ -82,4 +82,4 @@
}]
}
}
}

6
evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json
View File

@@ -10,10 +10,6 @@
"chrome"
],
"evaluator": {
"func": "infeasible",
"result": {
},
"expected": {
}
"func": "infeasible"
}
}

68
evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "59155008-fe71-45ec-8a8f-dc35497b6aa8",
"snapshot": "chrome",
"instruction": "What are the similar names to the name carl",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.babycenter.com/child"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.babycenter.com/baby-names/details/carl-853"
}
}
}
}

2
evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json
View File

@@ -18,7 +18,7 @@
{
"type": "execute",
"parameters": {
"command": "echo lixiaochuan20 | sudo -S apt-get update -y && echo lixiaochuan20 | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip",
"command": "echo password | sudo -S apt-get update -y && echo password | sudo -S apt-get install unzip -y && unzip /home/user/Desktop/helloExtension.zip -d /home/user/Desktop/ && rm /home/user/Desktop/helloExtension.zip",
"shell": true
}
},

6
evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json
View File

@@ -10,10 +10,6 @@
"chrome"
],
"evaluator": {
"func": "infeasible",
"result": {
},
"expected": {
}
"func": "infeasible"
}
}

2
evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json
View File

@@ -59,4 +59,4 @@
}
}
}
}

68
evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "a728a36e-8bf1-4bb6-9a03-ef039a5233f0",
"snapshot": "chrome",
"instruction": "Find the Driver License Eligibility Requirements",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.dmv.virginia.gov/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.dmv.virginia.gov/licenses-ids/license/applying/eligibility"
}
}
}
}

67
evaluation_examples/examples/chrome/a96b564e-dbe9-42c3-9ccf-b4498073438a.json
View File

@@ -68,6 +68,73 @@
"url": "https://discussions.flightaware.com/t/graphs-for-dump1090-my-version-with-install-script/46263"
}
}
=======
"id": "a96b564e-dbe9-42c3-9ccf-b4498073438a",
"snapshot": "chrome",
"instruction": "Find discussions of community and open one with most replies.",
"change_possibility": "low",
"source": "test_task_0",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.flightaware.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://discussions.flightaware.com/t/the-banter-thread/4412"
}
}
}
}

6
evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json
View File

@@ -10,10 +10,6 @@
"chrome"
],
"evaluator": {
"func": "infeasible",
"result": {
},
"expected": {
}
"func": "infeasible"
}
}

2
evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json
View File

@@ -62,4 +62,4 @@
}
}
}
}

4
evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json
View File

@@ -58,5 +58,5 @@
}
}
}
}
}
}

68
evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "f0b971a1-6831-4b9b-a50e-22a6e47f45ba",
"snapshot": "chrome",
"instruction": "Show me the scores for the 2019 super bowl",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.nfl.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.nfl.com/scores/2019/POST4"
}
}
}
}

68
evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "f5d96daf-83a8-4c86-9686-bada31fc66ab",
"snapshot": "chrome",
"instruction": "Compare iPhone 15 Pro Max with iPhone 13 Pro Max",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.apple.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.apple.com/iphone/compare/?modelList=iphone-15-pro-max,iphone-15-pro,iphone-13-pro-max"
}
}
}
}