lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Check and fix on Chrome tasks

Browse Source 
- Added `pytz` dependency to `requirements.txt` for timezone handling.
- Introduced `get_macys_product_url_parse` function to replace the old `get_url_path_parse` for better clarity and maintain backward compatibility.
- Enhanced logging throughout the `get_active_tab_html_parse` and `get_rule_relativeTime` functions for improved debugging and traceability.
- Updated JSON examples to reflect changes in expected keys and added new fields for better evaluation context.
- Removed deprecated execution commands from JSON examples to streamline the evaluation process.
This commit is contained in:
yuanmengqi
2025-07-06 07:52:37 +00:00
parent 1b40a458de
commit 9be6fcd688
20 changed files with 521 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
evaluation_examples/examples/chrome/0d8b7de3-e8de-4d86-b9fd-dd2dce58a217.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/chrome/121ba48f-9e17-48ce-9bc6-a4fb17a7ebba.json
View File

@@ -37,16 +37,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

4
evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json
View File

@@ -91,6 +91,7 @@
"from": "next Monday split",
"to": "next Friday split"
},
"timezone": "Europe/Zurich",
"expected": {
"puDay": "{DayD}",
"puMonth": "{MonthD}",
@@ -103,5 +104,6 @@
}
]
},
"proxy": true
"proxy": true,
"possibility_of_env_change": "medium"
}

13
evaluation_examples/examples/chrome/2888b4e6-5b47-4b57-8bf5-c73827890774.json
View File

@@ -1,7 +1,7 @@
{
"id": "2888b4e6-5b47-4b57-8bf5-c73827890774",
"snapshot": "chrome",
"instruction": "Show me all men's large-size short-sleeve T-shirts with a discount of 50% or more.",
"instruction": "Show me all men's large-size short-sleeve shirts with a discount of 50% or more.",
"source": "test_task_1",
"config": [
{
@@ -49,10 +49,10 @@
"goto_prefix": "https://www.",
"parse_keys": [
"mens_clothing",
"t_shirts",
"shirts",
"Men_regular_size_t",
"Price_discount_range",
"Sleeve_length"
"short_sleeve"
]
},
"expected": {
@@ -60,13 +60,14 @@
"rules": {
"expected": {
"mens_clothing": true,
"t_shirts": true,
"shirts": true,
"Men_regular_size_t": "L",
"Price_discount_range": "50_PERCENT_ off & more",
"Sleeve_length": "Short Sleeve"
"short_sleeve": true
}
}
}
},
"proxy": true
"proxy": false,
"possibility_of_env_change": "medium"
}

10
evaluation_examples/examples/chrome/59155008-fe71-45ec-8a8f-dc35497b6aa8.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

2
evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json
View File

@@ -63,6 +63,7 @@
"fT28tf":[
"Black",
"$25 - $60",
"On sale",
"is_other_exist"
]
}
@@ -84,6 +85,7 @@
"expected": {
"Black": true,
"$25 - $60": true,
"On sale": true,
"is_other_exist": false
}
}

10
evaluation_examples/examples/chrome/a728a36e-8bf1-4bb6-9a03-ef039a5233f0.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

10
evaluation_examples/examples/chrome/a96b564e-dbe9-42c3-9ccf-b4498073438a.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

3
evaluation_examples/examples/chrome/b4f95342-463e-4179-8c3f-193cd7241fb2.json
View File

@@ -62,5 +62,6 @@
"order": "2"
}
},
"proxy": true
"proxy": false,
"possibility_of_env_change": "high"
}

97
evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json
View File

@@ -1,7 +1,7 @@
{
"id": "b7895e80-f4d1-4648-bee0-4eb45a6f1fa8",
"snapshot": "chrome",
"instruction": "Find a Hotel in New York City with lowest price possible for 2 adults this weekend.",
"instruction": "Find a Hotel in New York City with lowest price possible for 2 adults next weekend.",
"source": "test_task_0",
"config": [
{
@@ -43,35 +43,74 @@
"chrome"
],
"evaluator": {
"func": "check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject": {
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[1]/div/button/div[3]": "from",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[2]/button/div[3]": "to",
"/html/body/div[1]/main/div[3]/div[2]/div/div[1]/div/h2": "city",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[3]/button/div[3]/span/span[2]": "adult",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[3]/div/div[2]/div/div/div[2]/div/button/div/div": "rank"
}
},
"expected": {
"type": "rule_relativeTime",
"rules": {
"relativeTime": {
"from": "this Saturday",
"to": "this Sunday"
},
"expected": {
"from": "{DoW}, {Month} {Day0D}",
"to": "{DoW}, {Month} {Day0D}",
"city": "New York City Hotels",
"adult": "2 guests",
"rank": "Price (low to high)"
"func": [
"check_direct_json_object",
"check_direct_json_object"
],
"conj": "or",
"result": [
{
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject": {
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[1]/div/button/div[3]": "from",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[2]/button/div[3]": "to",
"/html/body/div[1]/main/div[3]/div[2]/div/div[1]/div/h2": "city",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[3]/button/div[3]/span/span[2]": "adult",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[3]/div/div[2]/div/div/div[2]/div/button/div/div": "rank"
}
},
{
"type": "active_tab_html_parse",
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject": {
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[1]/div/button/div[3]": "from",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[2]/button/div[3]": "to",
"/html/body/div[1]/main/div[3]/div[2]/div/div[1]/div/h2": "city",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[1]/div/div/div/div[3]/button/div[3]/span/span[2]": "adult",
"/html/body/div[1]/main/div[3]/div[5]/div[2]/div/div[3]/div/div[2]/div/div/div[2]/div/button/div/div": "rank"
}
}
}
],
"expected": [
{
"type": "rule_relativeTime",
"rules": {
"relativeTime": {
"from": "next week Saturday",
"to": "next week Sunday"
},
"timezone": "America/New_York",
"expected": {
"from": "{DoW}, {Month} {Day0D}",
"to": "{DoW}, {Month} {Day0D}",
"city": "New York City Hotels",
"adult": "2 guests",
"rank": "Price (low to high)"
}
}
},
{
"type": "rule_relativeTime",
"rules": {
"relativeTime": {
"from": "next week Friday",
"to": "next week Sunday"
},
"timezone": "America/New_York",
"expected": {
"from": "{DoW}, {Month} {Day0D}",
"to": "{DoW}, {Month} {Day0D}",
"city": "New York City Hotels",
"adult": "2 guests",
"rank": "Price (low to high)"
}
}
}
]
},
"proxy": true
"proxy": true,
"possibility_of_env_change": "medium"
}

13
evaluation_examples/examples/chrome/c1fa57f3-c3db-4596-8f09-020701085416.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
@@ -67,5 +57,6 @@
}
}
},
"proxy": true
"proxy": true,
"possibility_of_env_change": "medium"
}

2
evaluation_examples/examples/chrome/da46d875-6b82-4681-9284-653b0c7ae241.json
View File

@@ -1,7 +1,7 @@
{
"id": "da46d875-6b82-4681-9284-653b0c7ae241",
"snapshot": "chrome",
"instruction": "Book an appointment to apply for a transportation access pass at the Charlie Card store on the first Monday eight months later, 10:15 am, fill in my details (James Smith, james.smith@gmail.com). And don not click \"book\" directly. Let me review it.",
"instruction": "Book an appointment to apply for a transportation access pass at the Charlie Card store on the first Monday eight months later, 10:15 am, fill in my details (James Smith, james.smith@gmail.com). And do not click \"book\" directly. Let me review it.",
"source": "test_task_2",
"config": [
{

10
evaluation_examples/examples/chrome/f0b971a1-6831-4b9b-a50e-22a6e47f45ba.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",

13
evaluation_examples/examples/chrome/f5d96daf-83a8-4c86-9686-bada31fc66ab.json
View File

@@ -36,16 +36,6 @@
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
@@ -76,5 +66,6 @@
}
}
},
"proxy": true
"proxy": true,
"possibility_of_env_change": "medium"
}