lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/xlang-ai/DesktopEnv

Browse Source
This commit is contained in:
tsuky_chen
2024-02-28 21:16:29 +08:00
parent 47b394dfc1 b4d066d8a5
commit c3c2b3c399
72 changed files with 3303 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation_examples/examples/chrome/030eeff7-b492-4218-b312-701ec99ee0cc.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
"--remote-debugging-port=1337"
]
}
},

69
evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "0d8b7de3-e8de-4d86-b9fd-dd2dce58a217",
"snapshot": "chrome",
"instruction": "Browse the natural products database.",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://drugs.com"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.drugs.com/npc/"
}
}
}
}

3
evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
View File

@@ -30,7 +30,8 @@
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
"type": "active_url_from_accessTree",
"goto_prefix": ""
},
"expected": {
"type": "rule",

71
evaluation_examples/examples/chrome/121ba48f-9e17-48ce-9bc6-a4fb17a7ebba.json Normal file
View File

@@ -0,0 +1,71 @@
{
"id": "121ba48f-9e17-48ce-9bc6-a4fb17a7ebba",
"snapshot": "chrome",
"instruction": "Find Dota 2 game and add all DLC to cart.",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.dota2.com/home",
"https://store.steampowered.com"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_added_to_steam_cart",
"result": {
"type": "page_info",
"url": "https://store.steampowered.com/cart/"
},
"expected": {
"type": "rule",
"rules": {
"items": [
"The Dota 2 Official Soundtrack"
]
}
}
}
}

87
evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json Normal file
View File

@@ -0,0 +1,87 @@
{
"id": "1704f00f-79e6-43a7-961b-cedd3724d5fd",
"snapshot": "chrome",
"instruction": "Find a large car with lowest price from next Monday to next Friday in Zurich.",
"source": "test_task_0",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.rentalcars.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":["check_direct_json_object", "check_direct_json_object"],
"result": [{
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": ["locationName", "dropLocationName", "filterCriteria_carCategory", "filterCriteria_sortBy"]
},
{
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject":{
"/html/body/main/div/div/div/section/div/div/div/div[1]/div[1]/p": "from",
"/html/body/main/div/div/div/section/div/div/div/div[1]/div[3]/p": "to"
}
}],
"expected":[{
"type": "rule",
"rules":{
"expected": {
"locationName": "Zurich Airport",
"dropLocationName": "Zurich Airport",
"filterCriteria_carCategory": "large",
"filterCriteria_sortBy": "PRICE"
}
}
},
{
"type": "rule_relativeTime",
"rules":{
"relativeTime":{
"from":"next Monday",
"to":"next Friday"
},
"expected": {
"from": "{DoW}, {DayD} {Month} {Year}, 10:00",
"to": "{DoW}, {DayD} {Month} {Year}, 10:00"
}
}}
]
}
}

63
evaluation_examples/examples/chrome/2888b4e6-5b47-4b57-8bf5-c73827890774.json Normal file
View File

@@ -0,0 +1,63 @@
{
"id": "2888b4e6-5b47-4b57-8bf5-c73827890774",
"snapshot": "chrome",
"instruction": "Find a men's T-Shirt that is in large size with a stripe pattern, short sleeve and under the Sales&Discount.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.macys.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"exact_match",
"result": {
"type": "url_dashPart",
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": true,
"returnType": "string"
},
"expected":{
"type": "rule",
"rules":{
"expected": "Stripe,Men,L,Short%20Sleeve,Sales%20%26%20Discounts"
}
}
}
}

2
evaluation_examples/examples/chrome/3299584d-8f11-4457-bf4c-ce98f7600250.json
View File

@@ -23,7 +23,7 @@
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
"--remote-debugging-port=1337"
]
}
},

78
evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json Normal file
View File

@@ -0,0 +1,78 @@
{
"id": "368d9ba4-203c-40c1-9fa3-da2f1430ce63",
"snapshot": "chrome",
"instruction": "find the Monthly forecast for Manchester, GB for this month",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.accuweather.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":["check_direct_json_object", "is_expected_url_pattern_match"],
"result": [{
"type": "url_dashPart",
"goto_prefix": "https://www.",
"partIndex": -2,
"needDeleteId": false,
"returnType": "json",
"key":"time"
},
{
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
}],
"expected":[
{
"type": "rule_relativeTime",
"rules":{
"relativeTime": {
"from": "this month"
},
"expected": {
"time": "{month}-weather"
}
}
},
{
"type": "rule",
"rules":{
"expected": ["\/manchester\/"]
}
}]
}
}

111
evaluation_examples/examples/chrome/47543840-672a-467d-80df-8f7c3b9788c9.json Normal file
View File

@@ -0,0 +1,111 @@
{
"id": "47543840-672a-467d-80df-8f7c3b9788c9",
"snapshot": "chrome",
"instruction": "Find and select the car with the most number of seats to pick up in Boston Logan Intl Airport from 10th next month to 11th next month.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.budget.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":["is_expected_url_pattern_match", "check_direct_json_object", "check_direct_json_object"],
"conj": "and",
"result": [
{
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
{
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject":{},
"class_multiObject":{
"location-info":{
"0": "start_location",
"1": "end_location"
},
"day-time-info":{
"0": "from",
"1": "to"
}
}
},
{
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject":{
"/html/body/div[6]/div[2]/div[1]/div/div/div[2]/div[1]/section[1]/div/form/div[1]/div[2]/div/a": "rank"
}
}
],
"expected":[
{
"type": "rule",
"rules":{
"expected": ["reservation#\/vehicles"]
}
},
{
"type": "rule_relativeTime",
"rules":{
"relativeTime":{
"from":"10th next month",
"to": "11th next month"
},
"expected": {
"start_location": "Boston Logan Intl Airport,\n\t\t\t\t\t\t\t\tBOS \n\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t\t\n\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t \n\t\t\t\t\t\t\t Pick-Up",
"end_location": "Boston Logan Intl Airport,\n\t\t\t\t\t\t\t\tBOS",
"from": "{DoW}, {Month} {Day0D}, 12:00 PM",
"to": "{DoW}, {Month} {Day0D}, 12:00 PM"
}
}
},
{
"type": "rule",
"rules":{
"expected": {
"rank": "Number of Seats (High to Low)"
}
}
}
]
}
}

69
evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "59155008-fe71-45ec-8a8f-dc35497b6aa8",
"snapshot": "chrome",
"instruction": "What are the similar names to the name carl",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.babycenter.com/child"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.babycenter.com/baby-names/details/carl-853"
}
}
}
}

2
evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json
View File

@@ -27,7 +27,7 @@
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
"--remote-debugging-port=1337"
]
}
},

78
evaluation_examples/examples/chrome/6c4c23a1-42a4-43cc-9db1-2f86ff3738cc.json Normal file
View File

@@ -0,0 +1,78 @@
{
"id": "6c4c23a1-42a4-43cc-9db1-2f86ff3738cc",
"snapshot": "chrome",
"instruction": "Find flights from Seattle to New York on 5th next month and only show those that can be purchased with miles.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.delta.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject":{
"search-date": "time",
"price-in-tabs__nav--selected": "category"
},
"class_multiObject":{
"search-segment-cities__city": {
"0": "start",
"1": "end"
}
}
},
"expected":{
"type": "rule_relativeTime",
"rules":{
"relativeTime": {
"from": "5th next month"
},
"expected": {
"start": "SEA",
"end": "NYC",
"time": "{DoW}, {Month} {DayD}, {Year}",
"category": "Miles"
}
}
}
}
}

63
evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json Normal file
View File

@@ -0,0 +1,63 @@
{
"id": "7f52cab9-535c-4835-ac8c-391ee64dc930",
"snapshot": "chrome",
"instruction": "Create a list of drip coffee makers that are on sale and within $25-60 and have a black finish.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://shopping.google.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": ["q", "tbs"]
},
"expected":{
"type": "rule",
"rules":{
"expected": {
"q": "drip coffee maker",
"tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,pdtr0:1825161|1825162"
}
}
}
}
}

65
evaluation_examples/examples/chrome/82279c77-8fc6-46f6-9622-3ba96f61b477.json Normal file
View File

@@ -0,0 +1,65 @@
{
"id": "82279c77-8fc6-46f6-9622-3ba96f61b477",
"snapshot": "chrome",
"instruction": "Find electric cars with a maximum price of $50,000 within 50 miles of 10001.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.cars.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": ["list_price_max", "maximum_distance", "zip","fuel_slugs[]"]
},
"expected":{
"type": "rule",
"rules":{
"expected": {
"list_price_max": "50000",
"maximum_distance": "50",
"zip":"10001",
"fuel_slugs[]":"electric"
}
}
}
}
}

69
evaluation_examples/examples/chrome/82bc8d6a-36eb-4d2d-8801-ef714fb1e55a.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "82bc8d6a-36eb-4d2d-8801-ef714fb1e55a",
"snapshot": "chrome",
"instruction": "On next Monday, look up a flight from Mumbai to Stockholm.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.qatarairways.com/en-hk/homepage.html"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": ["fromStation", "toStation", "departing"],
"replace":{
"departing": "time"
}
},
"expected":{
"type": "rule_relativeTime",
"rules":{
"relativeTime":{
"from": "next Monday"
},
"expected": {
"fromStation": "BOM",
"toStation": "STO",
"time": "{Year}-{Month0D}-{Day0D}"
}
}
}
}
}

2
evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
"--remote-debugging-port=1337"
]
}
},

29
evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json
View File

@@ -3,16 +3,41 @@
"snapshot": "chrome",
"instruction": "Please help me set Chrome to delete my browsing history automatically every time I close the browser.",
"source": "https://www.youtube.com/watch?v=v0kxqB7Xa6I",
"config": [],
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "",
"func": "exact_match",
"result": {
"type": "data_delete_automacally"
},
"expected": {
"type": "rule",
"rules": {
"expected": "Crashed"
}
}
}
}

78
evaluation_examples/examples/chrome/9f3f70fc-5afc-4958-a7b7-3bb4fcb01805.json Normal file
View File

@@ -0,0 +1,78 @@
{
"id": "9f3f70fc-5afc-4958-a7b7-3bb4fcb01805",
"snapshot": "chrome",
"instruction": "Browse the list of women's Nike jerseys over $60.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.nba.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":["is_expected_url_pattern_match", "check_direct_json_object"],
"conj": "and",
"result": [
{
"type": "active_tab_info"
},
{
"type": "active_tab_html_parse",
"category": "xpath",
"xpathObject":{
"/html/body/div[2]/div/div[6]/div[2]/div[2]/div/div[1]/div[4]/ul/li[2]": "money"
}
}
],
"expected":[
{
"type": "rule",
"rules":{
"expected": ["\/women-jerseys\/"]
}
},
{
"type": "rule",
"rules":{
"expected": {
"money": "over $60"
}
}
}
]
}
}

60
evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json Normal file
View File

@@ -0,0 +1,60 @@
{
"id": "9f935cce-0a9f-435f-8007-817732bfc0a5",
"snapshot": "chrome",
"instruction": "Browse list of Civil Division forms.",
"source": "online_tasks",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.justice.gov/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_url_pattern_match",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"expected": [
"forms\\?title=&field_component_target_id=431"
]
}
}
}
}

69
evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "a728a36e-8bf1-4bb6-9a03-ef039a5233f0",
"snapshot": "chrome",
"instruction": "Find the Driver License Eligibility Requirements",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.dmv.virginia.gov/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.dmv.virginia.gov/licenses-ids/license/applying/eligibility"
}
}
}
}

69
evaluation_examples/examples/chrome/a96b564e-dbe9-42c3-9ccf-b4498073438a.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "a96b564e-dbe9-42c3-9ccf-b4498073438a",
"snapshot": "chrome",
"instruction": "Find discussions of community and open one with most replies.",
"source": "test_task_0",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.flightaware.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"is_expected_active_tab",
"result": {
"type": "active_tab_info",
"goto_prefix": "https://www."
},
"expected":{
"type": "rule",
"rules":{
"type": "url",
"url": "https://discussions.flightaware.com/t/graphs-for-dump1090-my-version-with-install-script/46263"
}
}
}
}

61
evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json Normal file
View File

@@ -0,0 +1,61 @@
{
"id": "b070486d-e161-459b-aa2b-ef442d973b92",
"snapshot": "chrome",
"instruction": "Show side effects of Tamiflu.",
"source": "online_tasks",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.drugs.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "exact_match",
"result": {
"type": "url_dashPart",
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": false,
"returnType": "string"
},
"expected": {
"type": "rule",
"rules": {
"expected": "tamiflu.html#side-effects"
}
}
}
}

67
evaluation_examples/examples/chrome/b4f95342-463e-4179-8c3f-193cd7241fb2.json Normal file
View File

@@ -0,0 +1,67 @@
{
"id": "b4f95342-463e-4179-8c3f-193cd7241fb2",
"snapshot": "chrome",
"instruction": "Find the next available date for Albion Basin.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.recreation.gov/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category":"class",
"class_singleObject":{},
"class_multiObject":{
"camp-sortable-column-header":{
"2":"camp-sortable-column-header"
}
}
},
"expected":{
"type":"gotoRecreationPage_and_get_html_content",
"selector": "class",
"class": "camp-sortable-column-header",
"order": "2"
}
}
}

78
evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json Normal file
View File

@@ -0,0 +1,78 @@
{
"id": "b7895e80-f4d1-4648-bee0-4eb45a6f1fa8",
"snapshot": "chrome",
"instruction": "Find a Hotel in New York City with lowest price possible for 2 adults this weekend.",
"source": "test_task_0",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.tripadvisor.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject":{
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[2]/div/div/div/div/div[1]/div/button/div[3]":"from",
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[2]/div/div/div/div/div[2]/button/div[3]":"to",
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[1]/div/h1":"city",
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[2]/div/div/div/div/div[3]/button/div[3]/span/span[2]":"adult",
"/html/body/div[1]/main/div[3]/div/div[2]/div/div[1]/div/div[2]/div[1]/div/div[1]/div/div[1]/div[2]/div/div[2]/div/button/div/div":"rank"
}
},
"expected":
{
"type": "rule_relativeTime",
"rules":{
"relativeTime": {
"from": "this Saturday",
"to": "this Sunday"
},
"expected": {
"from": "{DoW}, {Month} {Day0D}",
"to": "{DoW}, {Month} {Day0D}",
"city": "New York City Hotels",
"adult": "2 adults",
"rank": "Price (low to high)"
}
}
}
}
}

2
evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=9222"
"--remote-debugging-port=1337"
]
}
},

69
evaluation_examples/examples/chrome/c1fa57f3-c3db-4596-8f09-020701085416.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "c1fa57f3-c3db-4596-8f09-020701085416",
"snapshot": "chrome",
"instruction": "Open the baggage fee calculator.",
"source": "test_task_1",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.united.com/en/us"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"is_expected_url_pattern_match",
"result": {
"type": "active_tab_info",
"goto_prefix": "https://www."
},
"expected":{
"type": "rule",
"rules":{
"expected": ["baggage-calculator"]
}
}
}
}

62
evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json Normal file
View File

@@ -0,0 +1,62 @@
{
"id": "cabb3bae-cccb-41bd-9f5d-0f3a9fecd825",
"snapshot": "chrome",
"instruction": "Browse spider-man toys for kids and sort by lowest price.",
"source": "online_tasks",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.kohls.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_url_pattern_match",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"expected": [
"AgeAppropriate:Kids",
"search=spider-man%20toys",
"S=4"
]
}
}
}
}

102
evaluation_examples/examples/chrome/da46d875-6b82-4681-9284-653b0c7ae241.json Normal file
View File

@@ -0,0 +1,102 @@
{
"id": "da46d875-6b82-4681-9284-653b0c7ae241",
"snapshot": "chrome",
"instruction": "Schedule an appointment to apply for transportation access pass in the Charlie Card store on the first Monday four months later, 10:15 am, fill in my details (James Smith, james.smith@gmail.com). And don not click \"book\" directly. Let me review it.",
"source": "test_task_2",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.mbta.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":["is_expected_url_pattern_match", "check_direct_json_object", "check_direct_json_object"],
"conj": "and",
"result": [
{
"type": "active_tab_info"
},
{
"type": "active_tab_html_parse",
"category": "class",
"class_singleObject":{},
"class_multiObject":{
"breakword":{
"1": "content",
"2": "time"
}
}
},
{
"type": "active_tab_html_parse",
"category": "input",
"inputObject":{
"/html/body/div/div/form/div[7]/div/div/div[1]/input[1]": "name",
"/html/body/div/div/form/div[7]/div/div/div[1]/input[2]": "mail"
}
}
],
"expected":[
{
"type": "rule",
"rules":{
"expected": ["CharlieCardStoreAppointments@mbta.com\/bookings\/"]
}
},
{
"type": "rule_relativeTime",
"rules":{
"relativeTime":{
"from":"first monday four months later"
},
"expected": {
"content": "Apply for Transportation Access Pass (TAP) CharlieCard non-auto approval",
"time": "{MonthFull} {Day0D}, 10:15 am"
}
}
},
{
"type": "rule",
"rules":{
"expected": {
"name": "James Smith",
"mail": "james.smith@gmail.com"
}
}
}
]
}
}

69
evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "f0b971a1-6831-4b9b-a50e-22a6e47f45ba",
"snapshot": "chrome",
"instruction": "Show me the scores for the 2019 super bowl",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.nfl.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.nfl.com/scores/2019/POST4"
}
}
}
}

59
evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json Normal file
View File

@@ -0,0 +1,59 @@
{
"id": "f3b19d1e-2d48-44e9-b4e1-defcae1a0197",
"snapshot": "chrome",
"instruction": "Find help page about buying tickets.",
"source": "test_task_0",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://seatgeek.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"is_expected_url_pattern_match",
"result": {
"type": "active_tab_info",
"goto_prefix": "https://www."
},
"expected":{
"type": "rule",
"rules":{
"expected": ["Buying-Tickets"]
}
}
}
}

69
evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "f5d96daf-83a8-4c86-9686-bada31fc66ab",
"snapshot": "chrome",
"instruction": "Compare iPhone 15 Pro Max with iPhone 13 Pro Max",
"source": "Mind2Web",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.apple.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.apple.com/iphone/compare/?modelList=iphone-15-pro-max,iphone-15-pro,iphone-13-pro-max"
}
}
}
}

74
evaluation_examples/examples/chrome/f79439ad-3ee8-4f99-a518-0eb60e5652b0.json Normal file
View File

@@ -0,0 +1,74 @@
{
"id": "f79439ad-3ee8-4f99-a518-0eb60e5652b0",
"snapshot": "chrome",
"instruction": "Search for a one way flight from Dublin to Vienna on 10th next month for 2 adults.",
"source": "test_task_2",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.ryanair.com/gb/en"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys":["originIata", "destinationIata", "tpAdults", "tpTeens", "tpChildren", "tpStartDate", "isReturn"],
"replace":{
"tpStartDate": "time"
}
},
"expected":{
"type": "rule_relativeTime",
"rules":{
"relativeTime": {
"from": "10th next month"
},
"expected": {
"originIata": "DUB",
"destinationIata": "VIE",
"tpAdults": "2",
"tpTeens": "0",
"tpChildren": "0",
"time": "{Year}-{Month0D}-{DayD}",
"isReturn":"false"
}
}
}
}
}

76
evaluation_examples/examples/chrome/fc6d8143-9452-4171-9459-7f515143419a.json Normal file
View File

@@ -0,0 +1,76 @@
{
"id": "fc6d8143-9452-4171-9459-7f515143419a",
"snapshot": "chrome",
"instruction": "Find the status of tomorrow flights from New York airports to Columbus in Ohio.",
"source": "test_task_0",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.delta.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject":{
"search-date": "time"
},
"class_multiObject":{
"search-segment-cities__city": {
"0": "start",
"1": "end"
}
}
},
"expected":{
"type": "rule_relativeTime",
"rules":{
"relativeTime": {
"from": "tomorrow"
},
"expected": {
"start": "NYC",
"end": "CMH",
"time": "{DoW}, {Month} {DayD}, {Year}"
}
}
}
}
}

39
evaluation_examples/examples/gimp/045bf3ff-9077-4b86-b483-a1040a949cff.json Normal file
View File

@@ -0,0 +1,39 @@
{
"id": "045bf3ff-9077-4b86-b483-a1040a949cff",
"snapshot": "gimp",
"instruction": "Could you turn my image into CYMK mode?",
"source": "https://www.makeuseof.com/tag/can-photoshop-gimp-cant/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1YnUq4q9rv7pr1MZPdaD2mVFZRrsgISmn",
"path": "/home/user/Desktop/gate.jpeg"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/gate.jpeg"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "infeasible",
"expected": {
},
"result": {
}
}
}

46
evaluation_examples/examples/gimp/2e6f678f-472d-4c55-99cc-8e7c5c402a71.json Normal file
View File

@@ -0,0 +1,46 @@
{
"id": "2e6f678f-472d-4c55-99cc-8e7c5c402a71",
"snapshot": "gimp",
"instruction": "Please batch process the images on the desktop by lifting the brightness to 50.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1uOZWtT9E8YW_IOu51meW5a0jAgwS1DoX",
"path": "/home/user/Desktop/squirrel.jpeg"
},
{
"url": "https://drive.google.com/uc?export=download&id=1KCyoqh3bTsbY42r9YSqIvcGuUr6i95GU",
"path": "/home/user/Desktop/panda.jpeg"
},
{
"url": "https://drive.google.com/uc?export=download&id=1xftsr0mRUvqKGPCHOnzUMm7tMnuqdhAA",
"path": "/home/user/Desktop/heron.jpeg"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "infeasible",
"expected": {
},
"result": {
}
}
}

39
evaluation_examples/examples/gimp/58d3eeeb-e9d0-499f-962e-fd0db2a744d8.json Normal file
View File

@@ -0,0 +1,39 @@
{
"id": "58d3eeeb-e9d0-499f-962e-fd0db2a744d8",
"snapshot": "gimp",
"instruction": "Please describe what is happening in the image.",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1w9R8RYkbFl88n8_Vgk_zZeD5J4t4yrzC",
"path": "/home/user/Desktop/heron.jpeg"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/heron.jpeg"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "infeasible",
"expected": {
},
"result": {
}
}
}

39
evaluation_examples/examples/gimp/8ea73f6f-9689-42ad-8c60-195bbf06a7ba.json Normal file
View File

@@ -0,0 +1,39 @@
{
"id": "8ea73f6f-9689-42ad-8c60-195bbf06a7ba",
"snapshot": "gimp",
"instruction": "Can you enhance this low-resolution photo to high-resolution without increasing the file size?",
"source": "",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=14r8Y0Cq8yTtQ5kfqhwWij5ow3LWApy2n",
"path": "/home/user/Desktop/low_resolution.jpeg"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/low_resolution.jpeg"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "infeasible",
"expected": {
},
"result": {
}
}
}

39
evaluation_examples/examples/gimp/dbbf4b99-2253-4b10-9274-45f246af2466.json Normal file
View File

@@ -0,0 +1,39 @@
{
"id": "dbbf4b99-2253-4b10-9274-45f246af2466",
"snapshot": "gimp",
"instruction": "Convert my new raw image into jpeg.",
"source": "https://www.makeuseof.com/tag/can-photoshop-gimp-cant/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1zcxzbQ5dp-mgpu0mom-LB8BKJH-V01Kr",
"path": "/home/user/Desktop/yicun.raw"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"/home/user/Desktop/yicun.raw"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "infeasible",
"expected": {
},
"result": {
}
}
}

2
evaluation_examples/examples/libreoffice_impress/73c99fb9-f828-43ce-b87a-01dc07faa224.json
View File

@@ -1,7 +1,7 @@
{
"id": "73c99fb9-f828-43ce-b87a-01dc07faa224",
"snapshot": "libreoffice_impress",
"instruction": "Add \"Page 1\" into the slide.",
"instruction": "Add \"Page 1\" into the content textbox on Slide 2.",
"source": "https://arxiv.org/pdf/2311.01767.pdf",
"config": [
{

2
evaluation_examples/examples/libreoffice_impress/986fc832-6af2-417c-8845-9272b3a1528b.json
View File

@@ -1,7 +1,7 @@
{
"id": "986fc832-6af2-417c-8845-9272b3a1528b",
"snapshot": "libreoffice_impress",
"instruction": "underline the content and make the font color on this slide dark red 2.",
"instruction": "underline the content and make the font color on this slide (including the table) dark red 2.",
"source": "https://arxiv.org/pdf/2311.01767.pdf",
"config": [
{

39
evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
View File

@@ -52,16 +52,33 @@
}
}
],
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/H2O_Factsheet_WA.docx",
"dest": "H2O_Factsheet_WA.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506",
"dest": "H2O_Factsheet_WA_Gold.docx"
}
"func": [
"compare_docx_files",
"compare_subscript_contains"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/H2O_Factsheet_WA.docx",
"dest": "H2O_Factsheet_WA.docx"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/H2O_Factsheet_WA.docx",
"dest": "H2O_Factsheet_WA.docx"
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506",
"dest": "H2O_Factsheet_WA_Gold.docx"
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1dM_FSTGDWxSW64VEth_wKMYNkvw0y_tq&export=download&authuser=0&confirm=t&uuid=342f41e2-f48f-41ff-8942-f7dfe5de1dba&at=APZUnTXHfskcX3tvmrSbzCOyQIgb:1704976694506",
"dest": "H2O_Factsheet_WA_Gold.docx"
}
]
}
}

3
evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json
View File

@@ -62,6 +62,9 @@
"type": "vm_file",
"path": "/home/user/Desktop/CCCH9003_Tutorial_guidelines.docx",
"dest": "CCCH9003_Tutorial_guidelines.docx"
},
"options": {
"ignore_blanks": false
}
}
}

16
evaluation_examples/examples/os/37887e8c-da15-4192-923c-08fa390a176d.json
View File

@@ -28,6 +28,22 @@
"command": "./setup.sh",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Terminal"
}
}
],
"trajectory": "trajectories/",

16
evaluation_examples/examples/os/4783cc41-c03c-4e1b-89b4-50658f642bd5.json
View File

@@ -20,6 +20,22 @@
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Terminal"
}
}
],
"trajectory": "trajectories/",

2
evaluation_examples/examples/os/4d117223-a354-47fb-8b45-62ab1390a95f.json
View File

@@ -36,7 +36,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
"import pyautogui; import time; time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
]
}
},

2
evaluation_examples/examples/os/5ced85fc-fa1a-4217-95fd-0fb530545ce2.json
View File

@@ -10,7 +10,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
"import pyautogui; import time; time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
]
}
},

2
evaluation_examples/examples/vs_code/276cc624-87ea-4f08-ab93-f770e3790175.json
View File

@@ -28,7 +28,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {
"expected": {
"editor.wordWrapColumn": 50
}
}

2
evaluation_examples/examples/vs_code/30167b0c-976b-491b-a9e5-7747607c6a3c.json
View File

@@ -50,7 +50,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {}
"expected": {}
}
},
"result": {

2
evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json
View File

@@ -86,7 +86,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": "project"
"expected": "project"
}
},
"result": {

2
evaluation_examples/examples/vs_code/6ed0a554-cbee-4b44-84ea-fd6c042f4fe1.json
View File

@@ -67,7 +67,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {
"expected": {
"folders": [
{"path": "project"},
{"path": "data1"},

2
evaluation_examples/examples/vs_code/70745df8-f2f5-42bd-8074-fbc10334fcc5.json
View File

@@ -50,7 +50,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {
"expected": {
"files.autoSave": "afterDelay",
"files.autoSaveDelay": 500
}

2
evaluation_examples/examples/vs_code/930fdb3b-11a8-46fe-9bac-577332e2640e.json
View File

@@ -28,7 +28,7 @@
"expected": {
"type": "rule",
"rules": {
"expect":
"expected":
{
"key": "ctrl+j",
"command": "workbench.action.focusActiveEditorGroup",

2
evaluation_examples/examples/vs_code/9439a27b-18ae-42d8-9778-5f68f891805e.json
View File

@@ -28,7 +28,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {"debug.focusEditorOnBreak": false}
"expected": {"debug.focusEditorOnBreak": false}
}
},
"result": {

18
evaluation_examples/examples/vs_code/982d12a5-beab-424f-8d38-d2a48429e511.json
View File

@@ -1,9 +1,16 @@
{
"id": "982d12a5-beab-424f-8d38-d2a48429e511",
"snapshot": "vscode",
"instruction": "Please help me change the color theme of VS Code to Dark.",
"instruction": "Please help me change the color theme of VS Code to Visual Studio Dark.",
"source": "https://www.youtube.com/watch?v=ORrELERGIHs",
"config": [
{
"type": "command",
"parameters": {
"command": "SETTINGS_PATH=\"$HOME/.config/Code/User/settings.json\"; [ ! -f \"$SETTINGS_PATH\" ] && echo '{\"workbench.colorTheme\": \"Red\"}' > \"$SETTINGS_PATH\" || (grep -q \"workbench.colorTheme\" \"$SETTINGS_PATH\" && sed -i 's/\"workbench.colorTheme\": \"[^\"]*\"/\"workbench.colorTheme\": \"Red\"/' \"$SETTINGS_PATH\" || jq '. + {\"workbench.colorTheme\": \"Red\"}' \"$SETTINGS_PATH\" > temp.json && mv temp.json \"$SETTINGS_PATH\")",
"shell": true
}
},
{
"type": "launch",
"parameters": {
@@ -28,14 +35,13 @@
"expected": {
"type": "rule",
"rules": {
"expect": "2"
"expected": "{\"workbench.colorTheme\": \"Visual Studio Dark\"}\n"
}
},
"result": {
"type": "vscode_config",
"vscode_extension_command": "GetColorTheme",
"path": "/home/user/GetColorTheme.txt",
"dest": "GetColorTheme.txt"
"type": "vm_file",
"path": "/home/user/.config/Code/User/settings.json",
"dest": "settings.json"
}
}
}

2
evaluation_examples/examples/vs_code/9d425400-e9b2-4424-9a4b-d4c7abac4140.json
View File

@@ -28,7 +28,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {"workbench.editor.wrapTabs": true}
"expected": {"workbench.editor.wrapTabs": true}
}
},
"result": {

2
evaluation_examples/examples/vs_code/c6bf789c-ba3a-4209-971d-b63abf0ab733.json
View File

@@ -50,7 +50,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {
"expected": {
"files.exclude": {
"**/__pycache__": true
}

2
evaluation_examples/examples/vs_code/e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2.json
View File

@@ -38,7 +38,7 @@
"expected": {
"type": "rule",
"rules": {
"expect": {
"expected": {
"python.analysis.diagnosticSeverityOverrides": {"reportMissingImports": "none"}
}
}

2
evaluation_examples/examples/vs_code/ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae.json
View File

@@ -30,7 +30,7 @@
"expected": {
"type": "rule",
"rules": {
"expect":
"expected":
{
"key": "ctrl+f",
"command": "-list.find",