lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source
This commit is contained in:
Timothyxxx
2024-03-08 20:39:20 +08:00
parent 6f0fe4f482 3761de4a05
commit 2b119d59b4
11 changed files with 96 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1g2lhX7CtuT6kc2Bmss0WU5zT1eXnnbEU&export=download&authuser=0&confirm=t&uuid=556ab51e-f4b7-4140-b38e-f93c0219f43a&at=APZUnTWsI26GRLwtp-r3_m9qNZCd:1709791386045",
"url": "https://drive.usercontent.google.com/download?id=1g2lhX7CtuT6kc2Bmss0WU5zT1eXnnbEU&export=download&authuser=0&confirm=t&uuid=37232f45-fedf-46c1-a292-f1080eb8c281&at=APZUnTV8C-zCWkPmRAc3BpiKz1uo:1709894768912",
"path": "/home/user/Desktop/rsc-ebook-collection-2023.xlsx"
}
]
@@ -40,14 +40,6 @@
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.qatarairways.com/en-hk/homepage.html"
]
}
},
{
"type": "activate_window",
"parameters": {
@@ -55,7 +47,7 @@
}
}
],
"trajectory": "trajectories/",
"trajectory": "trajectories/68a25bd4-59c7-4f4d-975e-da0c8509c848",
"related_apps": [
"libreoffice_calc",
"chrome"

9
evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json
View File

@@ -31,9 +31,9 @@
"vscode"
],
"evaluator": {
"func": "exact_match",
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"type": "vm_command_error",
"command": [
"python",
"-c",
@@ -43,7 +43,10 @@
"expected": {
"type": "rule",
"rules": {
"expected": ""
"include": [],
"exclude": [
"ModuleNotFoundError: No module named"
]
}
}
}

49
evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json
View File

@@ -34,13 +34,20 @@
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Projects"]
"command": [
"mkdir",
"-p",
"/home/user/Projects"
]
}
},
{
"type": "launch",
"parameters": {
"command": ["nautilus", "/home/user/Projects"]
"command": [
"nautilus",
"/home/user/Projects"
]
}
}
],
@@ -90,32 +97,52 @@
"rules": {
"expect": [
{
"key": ["name"],
"key": [
"name"
],
"method": "eq",
"ref": "happy-extension"
},
{
"key": ["version"],
"key": [
"version"
],
"method": "eq",
"ref": "0.0.1"
},
{
"key": ["background", "scripts"],
"key": [
"background",
"scripts"
],
"method": "eq",
"ref": ["background_script.js"]
"ref": [
"background_script.js"
]
},
{
"key": ["browser_action", "default_icon"],
"key": [
"browser_action",
"default_icon"
],
"method": "eq",
"ref": {"64": "icons/icon.png"}
"ref": {
"64": "icons/icon.png"
}
},
{
"key": ["browser_action", "default_popup"],
"key": [
"browser_action",
"default_popup"
],
"method": "eq",
"ref": "browserAction/index.html"
},
{
"key": ["browser_action", "default_title"],
"key": [
"browser_action",
"default_title"
],
"method": "eq",
"ref": "happy-extension"
}
@@ -144,4 +171,4 @@
}
]
}
}
}

6
evaluation_examples/examples/multi_apps/869de13e-bef9-4b91-ba51-f6708c40b096.json
View File

@@ -74,10 +74,6 @@
"url": "https://drive.usercontent.google.com/download?id=110FbG5m_XDDUpO3bvUZTPzw9y0ce1KB5&export=download&authuser=0&confirm=t&uuid=cbce5e69-d4dd-4508-8192-647d6da54fc0&at=APZUnTWxxAS53KTZJklxT6riooU6:1709357558943",
"path": "/home/user/Desktop/IA_Format.docx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1O1yd3Dm_nKEtknQYzDAHqPllv0BeQ2bp&export=download&authuser=0&confirm=t&uuid=636ee37e-4ee1-4415-ba71-73e42335c4bf&at=APZUnTW7tJhD6LG7CglaqI_y0Exu:1709357564381",
"path": "/home/user/Desktop/S1578219008703378.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1WA-QneFJF9JJ_KKs9ksZ8N7ggkhRIt6Y&export=download&authuser=0&confirm=t&uuid=3fbc694a-8470-4a8f-a6e9-9d64d94b3a5e&at=APZUnTXi4PtZ42zRYDj-VGbqH2Ts:1709793215203",
"path": "/home/user/Desktop/paper01.pdf"
@@ -164,7 +160,7 @@
{
"type": "rule",
"rules": {
"expected": "07-cluster-kMean (1).ppt\n2023_validation_7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx\nassignment_mark_frontpage.docx\ncco-return-to-school-survey-underlying-data-tables.xlsx\nDOC_2480903712718068684.pdf\nFamily Status Equality-Eng (Aug 2021).pdf\nIA_Format.docx\nS1578219008703378.pdf\n"
"expected": "07-cluster-kMean (1).ppt\n2023_validation_7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx\nassignment_mark_frontpage.docx\ncco-return-to-school-survey-underlying-data-tables.xlsx\nDOC_2480903712718068684.pdf\nFamily Status Equality-Eng (Aug 2021).pdf\nIA_Format.docx\n"
}
}
]

31
evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json
View File

@@ -30,20 +30,25 @@
"chrome"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"ls",
"/home/user/"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "instructor-embedding"
"postconfig": [
{
"type": "execute",
"parameters": {
"command": "cd /home/user && ls -R instructor-embedding/ > log.txt",
"shell": true
}
}
],
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1aWWgwZFhsT1ByRbZkofO1Hf9zWf2Df7S&export=download&authuser=0&confirm=t&uuid=b3643d9a-5e1f-4eaf-9a78-887aebda7cee&at=APZUnTU3vX5b_NQpZW8q3nlbIo7p:1709898581495",
"dest": "log_Gold.txt"
},
"result": {
"type": "vm_file",
"path": "/home/user/log.txt",
"dest": "log.txt"
}
}
}

13
evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json
View File

@@ -7,7 +7,11 @@
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Papers"]
"command": [
"mkdir",
"-p",
"/home/user/Documents/Papers"
]
}
},
{
@@ -36,7 +40,10 @@
{
"type": "execute",
"parameters": {
"command": ["nautilus", "/home/user/Documents/Papers"]
"command": [
"nautilus",
"/home/user/Documents/Papers"
]
}
}
],
@@ -102,4 +109,4 @@
]
}
}
}
}

8
evaluation_examples/examples/multi_apps/c7c1e4c3-9e92-4eba-a4b8-689953975ea4.json
View File

@@ -23,14 +23,6 @@
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.qatarairways.com/en-hk/homepage.html"
]
}
},
{
"type": "download",
"parameters": {

2
evaluation_examples/examples/multi_apps/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json
View File

@@ -1,7 +1,7 @@
{
"id": "da52d699-e8d2-4dc5-9191-a2199e0b6a9b",
"snapshot": "libreoffice_calc",
"instruction": "Examine the spreadsheet on the desktop, which contains a record of books read in 2022. Identify the book with the slowest reading pace, measured in words per day. I have an empty document named 'book_list_result.docx' on the desktop; please open it and record the title there.",
"instruction": "Examine the spreadsheet on the desktop, which contains a record of books read in 2022. Take the website https://howlongtoread.com/ as a reference to identify the book with the slowest reading pace, measured in words per day. I have an empty document named 'book_list_result.docx' on the desktop; please open it and record the title there.",
"source": "GAIA",
"config": [
{

6
evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -1,7 +1,7 @@
{
"id": "f918266a-b3e0-4914-865d-4faa564f1aef",
"snapshot": "vscode",
"instruction": "Please finalize the code and retrieve the output from the Python script 'calculator.py' located on the desktop and save it as 'log.txt' in the same directory as the Python file.",
"instruction": "Please complete the code and retrieve the output from the Python script 'calculator.py' located on the desktop and save it as 'log.txt' in the same directory as the Python file.",
"source": "GAIA",
"config": [
{
@@ -35,8 +35,8 @@
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-14AgA1nHNL22VD_3QtRzWaMjIBa3RvJ&export=download&authuser=0&confirm=t&uuid=91e2d5bc-a7f0-4abc-9aed-aa8b4956fb45&at=APZUnTU2TksSVngTw6WRJv8wpmoU:1709367652463",
"dest": "log.txt"
"path": "https://drive.usercontent.google.com/download?id=1-14AgA1nHNL22VD_3QtRzWaMjIBa3RvJ&export=download&authuser=0&confirm=t&uuid=6aa05bf1-4964-4f7b-8983-d28540b4053b&at=APZUnTXuJgDHIYA2FZl3A_OQJEOF:1709881263131",
"dest": "log_Gold.txt"
},
"result": {
"type": "vm_file",