lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source
This commit is contained in:
Timothyxxx
2024-03-06 15:06:42 +08:00
parent f21d685108 69ef653a7c
commit ef7e8942af
13 changed files with 667 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
evaluation_examples/examples/multi_apps/20236825-b5df-46e7-89bf-62e1d640a897.json Normal file
View File

@@ -0,0 +1,58 @@
{
"id": "20236825-b5df-46e7-89bf-62e1d640a897",
"snapshot": "vscode",
"instruction": "I am coding on my algorithm practice. The doc \"bubble_Sort_tutorial.docx\" is the document for it. Help me finish the function 'bubbleSort' in 'bubbleSort.py' on the Desktop save the output in 'res.txt' on Desktop.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1AQtZ8Hrf7WtyUtaHMtDN-UErKRXnW64d&export=download&authuser=0&confirm=t&uuid=bf7f2b4b-ecf9-4260-b74e-db0cd40b58ca&at=APZUnTVimJzbmwJ8-4E1lq9ipJf_:1709624149129",
"path": "/home/user/Desktop/bubbleSort.zip"
},
{
"url": "https://drive.usercontent.google.com/download?id=168ZHCnK6v5PEZ8G5M25ZUW9fICk4OlfE&export=download&authuser=0&confirm=t&uuid=3642df08-dc40-4d37-93a0-8532e3012fb0&at=APZUnTUP1OTlq0kIgqcj7YSWw6MB:1709622592489",
"path": "/home/user/Desktop/Bubble_Sort_tutorial.docx"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"unzip /home/user/Desktop/bubbleSort.zip -d /home/user/Desktop/ && rm -rf /home/user/Desktop/bubbleSort.zip"
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/Bubble_Sort_tutorial.docx"
}
}
],
"trajectory": "trajectories/20236825-b5df-46e7-89bf-62e1d640a897",
"related_apps": [
"vscode",
"libreoffice_writer",
"os"
],
"evaluator": {
"func": "compare_text_file",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1g2Trt9oxQyW_sx8aIztFA0zNsE4yNw2x&export=download&authuser=0&confirm=t&uuid=342751c4-54f1-4760-9326-e7388845ded0&at=APZUnTV5BcbaxIZrDglWbs84Oxln:1709623697315",
"dest": "res.txt"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/res.txt",
"dest": "res.txt"
}
}
}

69
evaluation_examples/examples/multi_apps/227d2f97-562b-4ccb-ae47-a5ec9e142fbb.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "227d2f97-562b-4ccb-ae47-a5ec9e142fbb",
"snapshot": "gimp",
"instruction": "I have my .xcf file saved on Desktop. Could you help me copy the image and paste it into a Libreoffice Writer file? Save it as 'image.docx' on the Desktop.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Desktop/QTdHniCqfJbBLJe3L3nijU-1200-80.xcf",
"url": "https://drive.usercontent.google.com/download?id=1BGoDOu9bYIG7Twj5dVTxRIWDUgWzzDtP&export=download&authuser=0&confirm=t&uuid=235d1bb2-37a0-4d96-82bf-a87f31d03cb4&at=APZUnTX53EdR1stASFS3OH5luAtB:1709617456061"
}
]
}
}
],
"trajectory": "trajectories/227d2f97-562b-4ccb-ae47-a5ec9e142fbb",
"related_apps": [
"libreoffice_writer",
"gimp",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "image.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1.0
}
}
],
"func": "compare_docx_images",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/image.docx",
"dest": "image.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=11kWQc1XFEqcIMuW0-NnZRSdv1199OmVI&export=download&authuser=0&confirm=t&uuid=694676fd-1ac9-4501-8acf-f48018494c7f&at=APZUnTV-koL51ka5dHum_HpGywv_:1709618406292",
"dest": "image.docx"
}
}
}

62
evaluation_examples/examples/multi_apps/48d05431-6cd5-4e76-82eb-12b60d823f7d.json Normal file
View File

@@ -0,0 +1,62 @@
{
"id": "48d05431-6cd5-4e76-82eb-12b60d823f7d",
"snapshot": "os",
"instruction": "When I ran \"conda install datasets\" in terminal, I got \"conda: command not found\". Could you help me solve it?",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Terminal"
}
}
],
"trajectory": "trajectories/48d05431-6cd5-4e76-82eb-12b60d823f7d",
"related_apps": [
"os",
"chrome"
],
"evaluator": {
"func": "is_in_list",
"result": {
"type": "vm_command_line",
"command": "conda list",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"expected": "packages in environment at"
}
}
}
}

69
evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "81c425f5-78f3-4771-afd6-3d2973825947",
"snapshot": "libreoffice_calc",
"instruction": "Could you help me transfer LibreOffice Calc data in this file to a LibreOffice Writer table? Save it as \"price.docx\" on the desktop.",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Desktop/OSP_Envelope_Price-List_2023_5000.xlsx",
"url": "https://drive.usercontent.google.com/download?id=1tXhQ1aG0JMbp3Yz643QRCmJBCHKE_trr&export=download&authuser=0&confirm=t&uuid=b4441d6c-48ce-4d49-a8cb-9239e8e20d1a&at=APZUnTU-Ba8Vx0N8e1nuomP_-4mm:1709556978086"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/OSP_Envelope_Price-List_2023_5000.xlsx"
}
}
],
"trajectory": "trajectories/81c425f5-78f3-4771-afd6-3d2973825947",
"related_apps": [
"libreoffice_writer",
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "price.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
]
}
}
],
"func": "compare_docx_tables",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/price.docx",
"dest": "price.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1KbdlQC0qSAYewG8QnZgParnSwv3s3dub&export=download&authuser=0&confirm=t&uuid=15dcc25c-8168-425e-96e1-fd27e0d6904b&at=APZUnTVho4ZrREHf9DC4rKwdIi3R:1709557117932",
"dest": "price.docx"
}
}
}

47
evaluation_examples/examples/multi_apps/8df7e444-8e06-4f93-8a1a-c5c974269d82.json Normal file
View File

@@ -0,0 +1,47 @@
{
"id": "8df7e444-8e06-4f93-8a1a-c5c974269d82",
"snapshot": "libreoffice_writer",
"instruction": "In the \"reminder.docx\" on Desktop is the submission instruction of our essay work. My essay is saved as docx file in /home/user. Please help me prepare the files for submission as required. ",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1wrCJd2f0xYnrHcj6KDzCe96X9JsN3VI3&export=download&authuser=0&confirm=t&uuid=277cb94d-1981-4f4d-b1ba-bceac8146001&at=APZUnTWKU5DBnr_6-_ZlEdsvhpCz:1709633482673",
"path": "/home/user/Recruitment_and_retention_of_health_professionals_across_Europe.docx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1vzawJthEhQHcK4cUF0W9QT4zaFywO6aH&export=download&authuser=0&confirm=t&uuid=0fbb6a50-f9c1-44d2-b430-3af738d5fadc&at=APZUnTWyfv-N5f-EjnF8ob-VfCsD:1709633450986",
"path": "/home/user/Desktop/reminder.docx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/reminder.docx"
}
}
],
"trajectory": "trajectories/8df7e444-8e06-4f93-8a1a-c5c974269d82",
"related_apps": [
"libreoffice_writer",
"os"
],
"evaluator": {
"func": "compare_zip_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1IKRu-dMFP4Aqzq5-4TOmOWVci0qvC27K&export=download&authuser=0&confirm=t&uuid=e2dabad2-5648-4bc3-a40f-f008089cd613&at=APZUnTVh5JD5nT3EvutwHIaSnJAT:1709633945616",
"dest": "Recruitment_and_retention_of_health_professionals_across_Europe.zip"
},
"result": {
"type": "vm_file",
"path": "/home/user/Recruitment_and_retention_of_health_professionals_across_Europe.zip",
"dest": "Recruitment_and_retention_of_health_professionals_across_Europe.zip"
}
}
}

47
evaluation_examples/examples/multi_apps/a503b07f-9119-456b-b75d-f5146737d24f.json Normal file
View File

@@ -0,0 +1,47 @@
{
"id": "a503b07f-9119-456b-b75d-f5146737d24f",
"snapshot": "os",
"instruction": "I am preparing for my paper reading report. Could you help me download this image at https://github.com/xlang-ai/OpenAgents/blob/main/pics/openagents_overview.png and convert it to PDF format. Save it as \"openagents_overview.pdf\" on desktop.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/a503b07f-9119-456b-b75d-f5146737d24f",
"related_apps": [
"os",
"chrome",
"gimp",
"pdf"
],
"evaluator": {
"func": "compare_pdfs",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1gD1odFNuLT6TP6rpAv_hot86pKcI5wY1&export=download&authuser=0&confirm=t&uuid=9d17c689-95d3-45e4-b093-0165de4045b4&at=APZUnTV9SssKQCoYyYeVsi8e9zcX:1709649491796",
"dest": "openagents_overview.pdf"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/openagents_overview.pdf",
"dest": "openagents_overview.pdf"
}
}
}

70
evaluation_examples/examples/multi_apps/aad10cd7-9337-4b62-b704-a857848cedf2.json Normal file
View File

@@ -0,0 +1,70 @@
{
"id": "aad10cd7-9337-4b62-b704-a857848cedf2",
"snapshot": "chrome",
"instruction": "I want to get a local file version of this blog on https://developer.apple.com/design/human-interface-guidelines/searching to make my own revision. Just keep the main content on this page, which starts from 'searching' and ends before 'resources'. Help me save this blog into \"notes.docx\" on the desktop.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/aad10cd7-9337-4b62-b704-a857848cedf2",
"related_apps": [
"os",
"chrome"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "notes.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}
],
"func": "compare_docx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1r2KJv0H3foo1WlWnArxdXnaew-yixNqL&export=download&authuser=0&confirm=t&uuid=633cc27c-d38b-4c45-907d-025341b4af1c&at=APZUnTV8AW5F_aLVooprdfgt-Q-Z:1709547335200",
"dest": "notes.docx"
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/notes.docx",
"dest": "notes.docx"
}
}
}

85
evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json Normal file
View File

@@ -0,0 +1,85 @@
{
"id": "b337d106-053f-4d37-8da0-7f9c4043a66b",
"snapshot": "os",
"instruction": "I've recently wanted to try using the Vim editor to edit code, but my Vim editor doesn't show relative line numbers by default. Please search the internet for a tutorial on how to display relative line numbers and make it the default setting for my local Vim.",
"source": "authors",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(0.5); pyautogui.click(960, 540); time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5)"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Terminal"
}
}
],
"trajectory": "trajectories/b337d106-053f-4d37-8da0-7f9c4043a66b",
"related_apps": [
"os",
"chrome"
],
"evaluator": {
"postconfig": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1CyhWjUS2oov4Fzc0VRwTh6LiS2Qu-T_8&export=download&authuser=0&confirm=t&uuid=9d0e2c62-895c-4bb3-a057-30cae60329ed&at=APZUnTVngSwARjYsWSmhSyHAqwID:1709647023362",
"path": "eval.sh"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": "chmod +x eval.sh",
"shell": true
}
}
],
"func": "check_include_exclude",
"result": {
"type": "vm_command_line",
"command": "bash eval.sh",
"shell": true
},
"expected": {
"type": "rule",
"rules": {
"include": [
"The File Has Set Number!"
],
"exclude": []
}
}
}
}

69
evaluation_examples/examples/multi_apps/bb83cab4-e5c7-42c7-a67b-e46068032b86.json Normal file
View File

@@ -0,0 +1,69 @@
{
"id": "bb83cab4-e5c7-42c7-a67b-e46068032b86",
"snapshot": "libreoffice_impress",
"instruction": "I'd like to save an Impress file as a document that I can edit in Writer. Just keep all the text in impress in \"script.docx\" on Desktop and I will do the reformatting by myself. Thanks!",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Desktop/Unlocking-the-Power-of-ChatGPT.pptx",
"url": "https://drive.usercontent.google.com/download?id=15JpEBadJeJXmkaMOMNHV5QjyVmuofQMB&export=download&authuser=0&confirm=t&uuid=308636e7-6d09-4b72-81df-cc791fc83d30&at=APZUnTV4I6P3uydAZJ-Ls1n_i8sK:1709558907727"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/Unlocking-the-Power-of-ChatGPT.pptx"
}
}
],
"trajectory": "trajectories/bb83cab4-e5c7-42c7-a67b-e46068032b86",
"related_apps": [
"libreoffice_writer",
"libreoffice_impress",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "script.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
]
}
}
],
"func": "compare_docx_tables",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/script.docx",
"dest": "script.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-Sol1W8S7Ybj-3KBJJarbcYUqS5wAQ1C&export=download&authuser=0&confirm=t&uuid=d967f546-b8f7-4ac2-b8fc-b1635f1cfbc4&at=APZUnTUazmbS2X3BSXDEQtJgobgf:1709559012053",
"dest": "script.docx"
}
}
}