lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update infeasible examples from Chrome and Calc

Browse Source
This commit is contained in:
Timothyxxx
2024-02-14 16:51:07 +08:00
parent 3f59ff46dc
commit 8d69eec68f
5 changed files with 60 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json Normal file
View File

@@ -0,0 +1,19 @@
{
"id": "3720f614-37fd-4d04-8a6b-76f54f8c222d",
"snapshot": "chrome",
"instruction": "I am more familiar with Korean as I am from Korea. I want to use chrome with my mother tongue. Could you help me change the Chrome interface language to Korean? ",
"source": "https://superuser.com/questions/984668/change-interface-language-of-chrome-to-english",
"config": [
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "infeasible",
"result": {
},
"expected": {
}
}
}

19
evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json Normal file
View File

@@ -0,0 +1,19 @@
{
"id": "93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9",
"snapshot": "chrome",
"instruction": "Could you assist me in turning off the dark mode feature in Google Chrome? I've noticed that while dark mode is great for reducing glare, it actually makes it more challenging for me to read text clearly, especially with my astigmatism.",
"source": "https://superuser.com/questions/1417973/how-to-disable-google-chrome-dark-mode",
"config": [
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "infeasible",
"result": {
},
"expected": {
}
}
}

26
evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json
View File

@@ -4,40 +4,16 @@
"instruction": "Could you please change the number of search results displayed on one page to 50? I find that having more results visible at once significantly enhances my research efficiency, as it reduces the need to constantly click through multiple pages. ",
"source": "https://support.google.com/chrome/thread/219988391/increase-search-results-per-page?hl=en",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "exact_match",
"func": "infeasible",
"result": {
"type": "number_of_search_results"
},
"expected": {
"type": "rule",
"rules": {
"expected": 50
}
}
}
}

66
evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json
View File

@@ -4,78 +4,16 @@
"instruction": "Make sparkline chart line by line",
"source": "https://www.youtube.com/shorts/L3Z-F1QTQFY",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351",
"path": "/home/user/OrderId_Month_Chart.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/OrderId_Month_Chart.xlsx"
}
}
],
"trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08",
"related_apps": [
"libreoffice calc"
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
}
],
"func": "compare_table",
"func": "infeasible",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094",
"dest": "OrderId_Month_Chart_gold.xlsx"
},
"result": {
"type": "vm_file",
"path": "/home/user/OrderId_Month_Chart.xlsx",
"dest": "OrderId_Month_Chart.xlsx"
},
"options": {
"rules": [
{
"type": "sparkline",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
},
{
"type": "sheet_data",
"sheet_idx0": 0,
"sheet_idx1": "EI0"
}
]
}
}
}

19
evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json Normal file
View File

@@ -0,0 +1,19 @@
{
"id": "7b802dad-6e0f-4204-9815-d4e3f57627d8",
"snapshot": "libreoffice_calc",
"instruction": "I would like to sort this table based on cell color, placing all the rows marked with pink at the beginning, while keeping their order among themselves unchanged.",
"source": "https://www.youtube.com/shorts/Of-lzeP1usE",
"config": [
],
"trajectory": "trajectories/7b802dad-6e0f-4204-9815-d4e3f57627d8",
"related_apps": [
"libreoffice_calc"
],
"evaluator": {
"func": "infeasible",
"expected": {
},
"result": {
}
}
}