lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix chrome tasks (#230)

Browse Source 
* fix chrome

* fix: fix proxy setup

* feat&fix: add proxy support in setup and remove hardcoded proxy from example

* fix tasks

* fix chrome finished

* fix

* clean chrome_fix code

* clean chrome_fix code

---------

Co-authored-by: adlsdztony <zzl0712@connect.hku.hk>
This commit is contained in:
Yuan Mengqi
2025-07-03 21:32:41 +08:00
committed by GitHub
parent bdaf37e0e5
commit b2fb8b4222
26 changed files with 444 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation_examples/examples/chrome/12086550-11c0-466b-b367-1d9e75b3910e.json
View File

@@ -29,7 +29,7 @@
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"func": "is_expected_active_tab_approximate",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": ""

30
evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json
View File

@@ -1,7 +1,7 @@
{
"id": "1704f00f-79e6-43a7-961b-cedd3724d5fd",
"snapshot": "chrome",
"instruction": "Find a large car with lowest price from next Monday to next Friday in Zurich.",
"instruction": "Find a large car from next Monday to Friday in Zurich, sorted by price.",
"source": "test_task_0",
"config": [
{
@@ -59,13 +59,16 @@
]
},
{
"type": "active_tab_html_parse",
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject": {
"/html/body/main/div/div/div/section/div/div/div/div[1]/div[1]/p": "from",
"/html/body/main/div/div/div/section/div/div/div/div[1]/div[3]/p": "to"
}
"parse_keys": [
"puDay",
"puMonth",
"puYear",
"doDay",
"doMonth",
"doYear"
]
}
],
"expected": [
@@ -77,6 +80,7 @@
"dropLocationName": "Zürich",
"filterCriteria_carCategory": "large",
"filterCriteria_sortBy": "PRICE"
}
}
},
@@ -84,12 +88,16 @@
"type": "rule_relativeTime",
"rules": {
"relativeTime": {
"from": "next Monday",
"to": "next Friday"
"from": "next Monday split",
"to": "next Friday split"
},
"expected": {
"from": "{DoW}, {DayD} {Month} {Year}, 10:00",
"to": "{DoW}, {DayD} {Month} {Year}, 10:00"
"puDay": "{DayD}",
"puMonth": "{MonthD}",
"puYear": "{Year}",
"doDay": "{DayD}",
"doMonth": "{MonthD}",
"doYear":"{Year}"
}
}
}

24
evaluation_examples/examples/chrome/2888b4e6-5b47-4b57-8bf5-c73827890774.json
View File

@@ -1,7 +1,7 @@
{
"id": "2888b4e6-5b47-4b57-8bf5-c73827890774",
"snapshot": "chrome",
"instruction": "Find a men's T-Shirt that is in large size with a stripe pattern, short sleeve and under the Sales&Discount.",
"instruction": "Show me all men's large-size short-sleeve T-shirts with a discount of 50% or more.",
"source": "test_task_1",
"config": [
{
@@ -43,18 +43,28 @@
"chrome"
],
"evaluator": {
"func": "exact_match",
"func": "check_direct_json_object",
"result": {
"type": "url_dashPart",
"type": "url_path_parse",
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": true,
"returnType": "string"
"parse_keys": [
"mens_clothing",
"t_shirts",
"Men_regular_size_t",
"Price_discount_range",
"Sleeve_length"
]
},
"expected": {
"type": "rule",
"rules": {
"expected": "Stripe,Men,L,Short%20Sleeve,Sales%20%26%20Discounts"
"expected": {
"mens_clothing": true,
"t_shirts": true,
"Men_regular_size_t": "L",
"Price_discount_range": "50_PERCENT_ off & more",
"Sleeve_length": "Short Sleeve"
}
}
}
},

4
evaluation_examples/examples/chrome/35253b65-1c19-4304-8aa4-6884b8218fc0.json
View File

@@ -44,8 +44,8 @@
"expected": {
"type": "rule",
"rules": {
"type": "name",
"name": "Play Puzzle Game 2048"
"type": "exec",
"exec": "/opt/google/chrome/google-chrome --profile-directory=Default --app-id=poahllcmmahlafplfhgjomkjmeblpapf"
}
}
},

4
evaluation_examples/examples/chrome/47543840-672a-467d-80df-8f7c3b9788c9.json
View File

@@ -1,7 +1,7 @@
{
"id": "47543840-672a-467d-80df-8f7c3b9788c9",
"snapshot": "chrome",
"instruction": "Find and select the car with the most number of seats to pick up in Boston Logan Intl Airport from 10th next month to 11th next month.",
"instruction": "Show me the cars available for pickup at Boston Logan Intl Airport from the 10th to the 11th of next month, sorted by the number of seats to find the largest capacity.",
"source": "test_task_1",
"config": [
{
@@ -75,7 +75,7 @@
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject": {
"/html/body/div[6]/div[2]/div[1]/div/div/div[2]/div[1]/section[1]/div/form/div[1]/div[2]/div/a": "rank"
"/html/body/div[6]/div[2]/div[1]/div/div/div[2]/section[1]/div[1]/form/div[1]/div[1]/div[2]/div/a": "rank"
}
}
],

2
evaluation_examples/examples/chrome/6766f2b8-8a72-417f-a9e5-56fcaa735837.json
View File

@@ -1,7 +1,7 @@
{
"id": "6766f2b8-8a72-417f-a9e5-56fcaa735837",
"snapshot": "chrome",
"instruction": "Could you help me install the unpacked extension at /home/user/Desktop/ in Chrome?",
"instruction": "Could you help me unzip the downloaded extension file from /home/user/Desktop/ to /home/user/Desktop/ and configure it in Chrome's extensions?",
"source": "https://support.google.com/chrome/thread/205881926/it-s-possible-to-load-unpacked-extension-automatically-in-chrome?hl=en",
"config": [
{

10
evaluation_examples/examples/chrome/6c4c23a1-42a4-43cc-9db1-2f86ff3738cc.json
View File

@@ -49,11 +49,11 @@
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject": {
"search-date": "time",
"price-in-tabs__nav--selected": "category"
"mach-flight-context-info__wrapper--date": "time",
"mach-global-tabs-small__wrapper__tab--active": "category"
},
"class_multiObject": {
"search-segment-cities__city": {
"class_multiObject_child": {
"mach-flight-context-info__wrapper__info--separator": {
"0": "start",
"1": "end"
}
@@ -68,7 +68,7 @@
"expected": {
"start": "SEA",
"end": "NYC",
"time": "{DoW}, {Month} {DayD}, {Year}",
"time": "{DoW}, {Month} {Day0D}, {Year}",
"category": "Miles"
}
}

60
evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json
View File

@@ -43,24 +43,52 @@
"chrome"
],
"evaluator": {
"func": "check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": [
"q",
"tbs"
]
},
"expected": {
"type": "rule",
"rules": {
"expected": {
"q": "drip coffee maker",
"tbs": "mr:1,price:1,ppr_min:25,ppr_max:60,sales:1,pdtr0:1825161|1825162"
"func": [
"check_direct_json_object",
"check_direct_json_object"
],
"result": [
{
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": [
"q"
]
},
{
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "class",
"class_multiObject_search_exist": {
"fT28tf":[
"Black",
"$25 - $60",
"is_other_exist"
]
}
}
}
],
"expected": [
{
"type": "rule",
"rules": {
"expected": {
"q": "drip coffee maker"
},
"expect_in_result": true
}
},
{
"type": "rule",
"rules": {
"expected": {
"Black": true,
"$25 - $60": true,
"is_other_exist": false
}
}
}
]
},
"proxy": true
}

60
evaluation_examples/examples/chrome/9f3f70fc-5afc-4958-a7b7-3bb4fcb01805.json
View File

@@ -43,41 +43,37 @@
"chrome"
],
"evaluator": {
"func": [
"is_expected_url_pattern_match",
"check_direct_json_object"
],
"conj": "and",
"result": [
{
"type": "active_tab_info"
"func": "check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"category": "class&url",
"class_multiObject": {
"filter-selector-link": [
"over $60",
"women",
"jerseys",
"nike"
]
},
{
"type": "active_tab_html_parse",
"category": "xpath",
"xpathObject": {
"/html/body/div[2]/div/div[6]/div[2]/div[2]/div/div[1]/div[4]/ul/li[2]": "money"
"url_include_expected": [
"over $60",
"women",
"jerseys",
"nike"
]
},
"expected": {
"type": "rule",
"rules": {
"expected": {
"over $60": true,
"women": true,
"jerseys": true,
"nike": true,
"is_other_exist": false
}
}
],
"expected": [
{
"type": "rule",
"rules": {
"expected": [
"/women-jerseys/"
]
}
},
{
"type": "rule",
"rules": {
"expected": {
"money": "over $60"
}
}
}
]
}
},
"proxy": true
}

2
evaluation_examples/examples/chrome/b4f95342-463e-4179-8c3f-193cd7241fb2.json
View File

@@ -1,7 +1,7 @@
{
"id": "b4f95342-463e-4179-8c3f-193cd7241fb2",
"snapshot": "chrome",
"instruction": "Find the next available date for Albion Basin.",
"instruction": "Find the next available date for Diamond.",
"source": "test_task_1",
"config": [
{

12
evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json
View File

@@ -49,11 +49,11 @@
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject": {
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[2]/div/div/div/div/div[1]/div/button/div[3]": "from",
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[2]/div/div/div/div/div[2]/button/div[3]": "to",
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[1]/div/h1": "city",
"/html/body/div[1]/main/div[3]/div/div[1]/div[2]/div[1]/div[2]/div/div/div/div/div[3]/button/div[3]/span/span[2]": "adult",
"/html/body/div[1]/main/div[3]/div/div[2]/div/div[1]/div/div[2]/div[1]/div/div[1]/div/div[1]/div[2]/div/div[2]/div/button/div/div": "rank"
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[1]/div/button/div[3]": "from",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[2]/button/div[3]": "to",
"/html/body/div[1]/main/div[3]/div[2]/div/div[1]/div/h2": "city",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[3]/button/div[3]/span/span[2]": "adult",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[3]/div/div[2]/div/div/div[2]/div/button/div/div": "rank"
}
},
"expected": {
@@ -67,7 +67,7 @@
"from": "{DoW}, {Month} {Day0D}",
"to": "{DoW}, {Month} {Day0D}",
"city": "New York City Hotels",
"adult": "2 adults",
"adult": "2 guests",
"rank": "Price (low to high)"
}
}

4
evaluation_examples/examples/chrome/c1fa57f3-c3db-4596-8f09-020701085416.json
View File

@@ -1,7 +1,7 @@
{
"id": "c1fa57f3-c3db-4596-8f09-020701085416",
"snapshot": "chrome",
"instruction": "Open the baggage fee calculator.",
"instruction": "Open the baggage fee calculator in United Airlines website.",
"source": "test_task_1",
"config": [
{
@@ -62,7 +62,7 @@
"type": "rule",
"rules": {
"expected": [
"checked-bag-fee-calculator"
"united.com/en/us/checked-bag-fee-calculator"
]
}
}

36
evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json
View File

@@ -43,19 +43,39 @@
"chrome"
],
"evaluator": {
"func": "is_expected_url_pattern_match",
"func": "check_direct_json_object",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "class&url",
"class_multiObject_li": {
"pmpSearch_breadcrumb": [
"Spider-Man",
"Toys",
"Kids"
],
"sbSelector": [
"Price Low-High"
]
},
"url_include_expected_multichoice": {
"Spider-Man": "Spider-Man",
"spiderman": "Spider-Man",
"Toys": "Toys",
"Kids": "Kids",
"S=4": "Price Low-High"
}
},
"expected": {
"type": "rule",
"rules": {
"expected": [
"AgeAppropriate:Kids",
"search=spider[-%20]?man%20toys",
"S=4"
]
"expected": {
"spider-man": true,
"toys": true,
"kids": true,
"price low-high": true,
"is_other_exist": false
}
}
}
},

21
evaluation_examples/examples/chrome/da46d875-6b82-4681-9284-653b0c7ae241.json
View File

@@ -1,7 +1,7 @@
{
"id": "da46d875-6b82-4681-9284-653b0c7ae241",
"snapshot": "chrome",
"instruction": "Schedule an appointment to apply for transportation access pass in the Charlie Card store on the first Monday four months later, 10:15 am, fill in my details (James Smith, james.smith@gmail.com). And don not click \"book\" directly. Let me review it.",
"instruction": "Book an appointment to apply for a transportation access pass at the Charlie Card store on the first Monday eight months later, 10:15 am, fill in my details (James Smith, james.smith@gmail.com). And don not click \"book\" directly. Let me review it.",
"source": "test_task_2",
"config": [
{
@@ -56,11 +56,10 @@
{
"type": "active_tab_html_parse",
"category": "class",
"class_singleObject": {},
"class_multiObject": {
"breakword": {
"1": "content",
"2": "time"
"class_multiObject_only_child": {
"HAZ16": {
"0": "content",
"1": "time"
}
}
},
@@ -68,8 +67,8 @@
"type": "active_tab_html_parse",
"category": "input",
"inputObject": {
"/html/body/div/div/form/div[7]/div/div/div[1]/input[1]": "name",
"/html/body/div/div/form/div[7]/div/div/div[1]/input[2]": "mail"
"/html/body/div[2]/div/form/div[7]/div/div/div[1]/input[1]": "name",
"/html/body/div[2]/div/form/div[7]/div/div/div[1]/input[2]": "mail"
}
}
],
@@ -78,7 +77,7 @@
"type": "rule",
"rules": {
"expected": [
"CharlieCardStoreAppointments@mbta.com/bookings/"
"book/CharlieCardStoreAppointments@mbta.com/"
]
}
},
@@ -86,11 +85,11 @@
"type": "rule_relativeTime",
"rules": {
"relativeTime": {
"from": "first monday four months later"
"from": "first monday eight months later"
},
"expected": {
"content": "Apply for Transportation Access Pass (TAP) CharlieCard non-auto approval",
"time": "{MonthFull} {Day0D}, 10:15 am"
"time": "{MonthFull} {Day0D}, 10:15 AM"
}
}
},

2
evaluation_examples/examples/chrome/e1e75309-3ddb-4d09-92ec-de869c928143.json
View File

@@ -1,7 +1,7 @@
{
"id": "e1e75309-3ddb-4d09-92ec-de869c928143",
"snapshot": "chrome",
"instruction": "Computer, can you turn the webpage I'm looking at into a PDF file and put it on my main screen, you know, the Desktop?",
"instruction": "Computer, can you turn the webpage I'm looking at into a PDF file, save it to my Desktop with the default filename and set the margins to none?",
"source": "https://in5stepstutorials.com/google-chrome/save-web-page-as-pdf-in-chrome.php",
"config": [
{

4
evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json
View File

@@ -1,7 +1,7 @@
{
"id": "f0b971a1-6831-4b9b-a50e-22a6e47f45ba",
"snapshot": "chrome",
"instruction": "Show me the scores for the 2019 super bowl",
"instruction": "Please help me find the score record for the 2019 Super Bowl in the NFL website.",
"source": "Mind2Web",
"config": [
{
@@ -62,7 +62,7 @@
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.nfl.com/scores/2019/POST4"
"url": "https://www.nfl.com/scores/2019/post4"
}
}
},

11
evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json
View File

@@ -1,7 +1,7 @@
{
"id": "f3b19d1e-2d48-44e9-b4e1-defcae1a0197",
"snapshot": "chrome",
"instruction": "Find help page about buying tickets.",
"instruction": "Find the FAQ page about ticket delivery.",
"source": "test_task_0",
"config": [
{
@@ -27,7 +27,7 @@
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://seatgeek.com/"
"https://premier.ticketek.com.au/"
]
}
},
@@ -45,17 +45,16 @@
"evaluator": {
"func": "is_expected_url_pattern_match",
"result": {
"type": "active_tab_info",
"goto_prefix": "https://www."
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"expected": [
"Buying-Tickets"
"Ticket-Delivery-FAQs"
]
}
}
},
"proxy": true
"proxy": false
}

22
evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json
View File

@@ -1,7 +1,7 @@
{
"id": "f5d96daf-83a8-4c86-9686-bada31fc66ab",
"snapshot": "chrome",
"instruction": "Compare iPhone 15 Pro Max with iPhone 13 Pro Max",
"instruction": "Compare iPhone 15 Pro Max with iPhone 14 Pro Max and iPhone 13 Pro Max",
"source": "Mind2Web",
"config": [
{
@@ -53,16 +53,26 @@
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"func": "check_direct_json_object",
"result": {
"type": "active_url_from_accessTree",
"goto_prefix": "https://www."
"type": "active_tab_url_parse",
"goto_prefix": "https://www.",
"parse_keys": [
"modelList"
],
"split_list": true
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://www.apple.com/iphone/compare/?modelList=iphone-15-pro-max,iphone-15-pro,iphone-13-pro-max"
"expected": {
"modelList": [
"iphone-15-pro-max",
"iphone-14-pro-max",
"iphone-13-pro-max"
]
},
"ignore_list_order": true
}
}
},

8
evaluation_examples/examples/chrome/fc6d8143-9452-4171-9459-7f515143419a.json
View File

@@ -49,10 +49,10 @@
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject": {
"search-date": "time"
"mach-flight-context-info__wrapper--date": "time"
},
"class_multiObject": {
"search-segment-cities__city": {
"class_multiObject_child": {
"mach-flight-context-info__wrapper__info--separator": {
"0": "start",
"1": "end"
}
@@ -67,7 +67,7 @@
"expected": {
"start": "NYC",
"end": "CMH",
"time": "{DoW}, {Month} {DayD}, {Year}"
"time": "{DoW}, {Month} {Day0D}, {Year}"
}
}
}