lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix multi apps

Browse Source
This commit is contained in:
tsuky_chen
2024-03-08 20:36:34 +08:00
parent 5abdf207a9
commit 4070b41fbd
11 changed files with 96 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1g2lhX7CtuT6kc2Bmss0WU5zT1eXnnbEU&export=download&authuser=0&confirm=t&uuid=556ab51e-f4b7-4140-b38e-f93c0219f43a&at=APZUnTWsI26GRLwtp-r3_m9qNZCd:1709791386045",
"url": "https://drive.usercontent.google.com/download?id=1g2lhX7CtuT6kc2Bmss0WU5zT1eXnnbEU&export=download&authuser=0&confirm=t&uuid=37232f45-fedf-46c1-a292-f1080eb8c281&at=APZUnTV8C-zCWkPmRAc3BpiKz1uo:1709894768912",
"path": "/home/user/Desktop/rsc-ebook-collection-2023.xlsx"
}
]
@@ -40,14 +40,6 @@
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.qatarairways.com/en-hk/homepage.html"
]
}
},
{
"type": "activate_window",
"parameters": {
@@ -55,7 +47,7 @@
}
}
],
"trajectory": "trajectories/",
"trajectory": "trajectories/68a25bd4-59c7-4f4d-975e-da0c8509c848",
"related_apps": [
"libreoffice_calc",
"chrome"

9
evaluation_examples/examples/multi_apps/69acbb55-d945-4927-a87b-8480e1a5bb7e.json
View File

@@ -31,9 +31,9 @@
"vscode"
],
"evaluator": {
"func": "exact_match",
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"type": "vm_command_error",
"command": [
"python",
"-c",
@@ -43,7 +43,10 @@
"expected": {
"type": "rule",
"rules": {
"expected": ""
"include": [],
"exclude": [
"ModuleNotFoundError: No module named"
]
}
}
}

49
evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json
View File

@@ -34,13 +34,20 @@
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Projects"]
"command": [
"mkdir",
"-p",
"/home/user/Projects"
]
}
},
{
"type": "launch",
"parameters": {
"command": ["nautilus", "/home/user/Projects"]
"command": [
"nautilus",
"/home/user/Projects"
]
}
}
],
@@ -90,32 +97,52 @@
"rules": {
"expect": [
{
"key": ["name"],
"key": [
"name"
],
"method": "eq",
"ref": "happy-extension"
},
{
"key": ["version"],
"key": [
"version"
],
"method": "eq",
"ref": "0.0.1"
},
{
"key": ["background", "scripts"],
"key": [
"background",
"scripts"
],
"method": "eq",
"ref": ["background_script.js"]
"ref": [
"background_script.js"
]
},
{
"key": ["browser_action", "default_icon"],
"key": [
"browser_action",
"default_icon"
],
"method": "eq",
"ref": {"64": "icons/icon.png"}
"ref": {
"64": "icons/icon.png"
}
},
{
"key": ["browser_action", "default_popup"],
"key": [
"browser_action",
"default_popup"
],
"method": "eq",
"ref": "browserAction/index.html"
},
{
"key": ["browser_action", "default_title"],
"key": [
"browser_action",
"default_title"
],
"method": "eq",
"ref": "happy-extension"
}
@@ -144,4 +171,4 @@
}
]
}
}
}

6
evaluation_examples/examples/multi_apps/869de13e-bef9-4b91-ba51-f6708c40b096.json
View File

@@ -74,10 +74,6 @@
"url": "https://drive.usercontent.google.com/download?id=110FbG5m_XDDUpO3bvUZTPzw9y0ce1KB5&export=download&authuser=0&confirm=t&uuid=cbce5e69-d4dd-4508-8192-647d6da54fc0&at=APZUnTWxxAS53KTZJklxT6riooU6:1709357558943",
"path": "/home/user/Desktop/IA_Format.docx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1O1yd3Dm_nKEtknQYzDAHqPllv0BeQ2bp&export=download&authuser=0&confirm=t&uuid=636ee37e-4ee1-4415-ba71-73e42335c4bf&at=APZUnTW7tJhD6LG7CglaqI_y0Exu:1709357564381",
"path": "/home/user/Desktop/S1578219008703378.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1WA-QneFJF9JJ_KKs9ksZ8N7ggkhRIt6Y&export=download&authuser=0&confirm=t&uuid=3fbc694a-8470-4a8f-a6e9-9d64d94b3a5e&at=APZUnTXi4PtZ42zRYDj-VGbqH2Ts:1709793215203",
"path": "/home/user/Desktop/paper01.pdf"
@@ -164,7 +160,7 @@
{
"type": "rule",
"rules": {
"expected": "07-cluster-kMean (1).ppt\n2023_validation_7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx\nassignment_mark_frontpage.docx\ncco-return-to-school-survey-underlying-data-tables.xlsx\nDOC_2480903712718068684.pdf\nFamily Status Equality-Eng (Aug 2021).pdf\nIA_Format.docx\nS1578219008703378.pdf\n"
"expected": "07-cluster-kMean (1).ppt\n2023_validation_7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx\nassignment_mark_frontpage.docx\ncco-return-to-school-survey-underlying-data-tables.xlsx\nDOC_2480903712718068684.pdf\nFamily Status Equality-Eng (Aug 2021).pdf\nIA_Format.docx\n"
}
}
]

31
evaluation_examples/examples/multi_apps/acb0f96b-e27c-44d8-b55f-7cb76609dfcd.json
View File

@@ -30,20 +30,25 @@
"chrome"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"ls",
"/home/user/"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "instructor-embedding"
"postconfig": [
{
"type": "execute",
"parameters": {
"command": "cd /home/user && ls -R instructor-embedding/ > log.txt",
"shell": true
}
}
],
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1aWWgwZFhsT1ByRbZkofO1Hf9zWf2Df7S&export=download&authuser=0&confirm=t&uuid=b3643d9a-5e1f-4eaf-9a78-887aebda7cee&at=APZUnTU3vX5b_NQpZW8q3nlbIo7p:1709898581495",
"dest": "log_Gold.txt"
},
"result": {
"type": "vm_file",
"path": "/home/user/log.txt",
"dest": "log.txt"
}
}
}

13
evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json
View File

@@ -7,7 +7,11 @@
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Papers"]
"command": [
"mkdir",
"-p",
"/home/user/Documents/Papers"
]
}
},
{
@@ -36,7 +40,10 @@
{
"type": "execute",
"parameters": {
"command": ["nautilus", "/home/user/Documents/Papers"]
"command": [
"nautilus",
"/home/user/Documents/Papers"
]
}
}
],
@@ -97,4 +104,4 @@
]
}
}
}
}

8
evaluation_examples/examples/multi_apps/c7c1e4c3-9e92-4eba-a4b8-689953975ea4.json
View File

@@ -23,14 +23,6 @@
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.qatarairways.com/en-hk/homepage.html"
]
}
},
{
"type": "download",
"parameters": {

2
evaluation_examples/examples/multi_apps/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json
View File

@@ -1,7 +1,7 @@
{
"id": "da52d699-e8d2-4dc5-9191-a2199e0b6a9b",
"snapshot": "libreoffice_calc",
"instruction": "Examine the spreadsheet on the desktop, which contains a record of books read in 2022. Identify the book with the slowest reading pace, measured in words per day. I have an empty document named 'book_list_result.docx' on the desktop; please open it and record the title there.",
"instruction": "Examine the spreadsheet on the desktop, which contains a record of books read in 2022. Take the website https://howlongtoread.com/ as a reference to identify the book with the slowest reading pace, measured in words per day. I have an empty document named 'book_list_result.docx' on the desktop; please open it and record the title there.",
"source": "GAIA",
"config": [
{

6
evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -1,7 +1,7 @@
{
"id": "f918266a-b3e0-4914-865d-4faa564f1aef",
"snapshot": "vscode",
"instruction": "Please finalize the code and retrieve the output from the Python script 'calculator.py' located on the desktop and save it as 'log.txt' in the same directory as the Python file.",
"instruction": "Please complete the code and retrieve the output from the Python script 'calculator.py' located on the desktop and save it as 'log.txt' in the same directory as the Python file.",
"source": "GAIA",
"config": [
{
@@ -35,8 +35,8 @@
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-14AgA1nHNL22VD_3QtRzWaMjIBa3RvJ&export=download&authuser=0&confirm=t&uuid=91e2d5bc-a7f0-4abc-9aed-aa8b4956fb45&at=APZUnTU2TksSVngTw6WRJv8wpmoU:1709367652463",
"dest": "log.txt"
"path": "https://drive.usercontent.google.com/download?id=1-14AgA1nHNL22VD_3QtRzWaMjIBa3RvJ&export=download&authuser=0&confirm=t&uuid=6aa05bf1-4964-4f7b-8983-d28540b4053b&at=APZUnTXuJgDHIYA2FZl3A_OQJEOF:1709881263131",
"dest": "log_Gold.txt"
},
"result": {
"type": "vm_file",