lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:xlang-ai/DesktopEnv

Browse Source
This commit is contained in:
Jason Lee
2024-03-10 14:50:17 +08:00
parent 775cef744f a12dfacbd7
commit 812be97a41
27 changed files with 173 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
evaluation_examples/examples/multi_apps/02ce9a50-7af2-47ed-8596-af0c230501f8.json
View File

@@ -54,7 +54,7 @@
"type": "rule",
"rules": {
"type": "text",
"text": "$ ls\n"
"text": " Ls"
}
}
}

2
evaluation_examples/examples/multi_apps/09a37c51-e625-49f4-a514-20a773797a8a.json
View File

@@ -37,7 +37,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1Ee1vNyG7gGpLKK2VlLfj6PxcmdkMdvqK&export=download&authuser=0&confirm=t&uuid=1f441c5d-b62d-4850-870f-8e8f113a4091&at=APZUnTWEvKSSkuGBWzen0S9L7aHP:1709727474803",
"dest": "pic.jpg"
"dest": "pic_Gold.jpg"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/20236825-b5df-46e7-89bf-62e1d640a897.json
View File

@@ -47,7 +47,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1g2Trt9oxQyW_sx8aIztFA0zNsE4yNw2x&export=download&authuser=0&confirm=t&uuid=342751c4-54f1-4760-9326-e7388845ded0&at=APZUnTV5BcbaxIZrDglWbs84Oxln:1709623697315",
"dest": "res.txt"
"dest": "res_Gold.txt"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/227d2f97-562b-4ccb-ae47-a5ec9e142fbb.json
View File

@@ -63,7 +63,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=11kWQc1XFEqcIMuW0-NnZRSdv1199OmVI&export=download&authuser=0&confirm=t&uuid=694676fd-1ac9-4501-8acf-f48018494c7f&at=APZUnTV-koL51ka5dHum_HpGywv_:1709618406292",
"dest": "image.docx"
"dest": "image_Gold.docx"
}
}
}

6
evaluation_examples/examples/multi_apps/47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5.json
View File

@@ -25,6 +25,12 @@
"path": "/home/user/Desktop/Robotic_Workshop_Infographics.pptx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 3
}
},
{
"type": "launch",
"parameters": {

2
evaluation_examples/examples/multi_apps/4c26e3f3-3a14-4d86-b44a-d3cedebbb487.json
View File

@@ -1,7 +1,7 @@
{
"id": "4c26e3f3-3a14-4d86-b44a-d3cedebbb487",
"snapshot": "libreoffice_impress",
"instruction": "I've noticed that the image on the second slide is too dim. Can you please enhance its brightness for me? Save the adjusted image on the Desktop and name it \"background\". Thank you!",
"instruction": "I've noticed that the image on the second slide is too dim. Can you please enhance its brightness for me? Save the adjusted image on the Desktop and name it \"background.png\". Thank you!",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{

11
evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json
View File

@@ -23,15 +23,6 @@
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://arxiv.org/abs/2005.14165",
"https://wallhaven.cc/"
]
}
},
{
"type": "download",
"parameters": {
@@ -105,7 +96,7 @@
"147",
"372",
"Deep learning",
"https://creativecoding.soe.ucsc.edu/courses/cs523/slides/week3/DeepLearning_LeCun.pdf"
"https://hal.science/hal-04206682/document"
]
},
"options": {

4
evaluation_examples/examples/multi_apps/68a25bd4-59c7-4f4d-975e-da0c8509c848.json
View File

@@ -61,12 +61,12 @@
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1kAp7ulAR_h4snny212yg1xyR1cMy3H2Q&export=download&authuser=0&confirm=t&uuid=3f6cb74c-63cc-4653-9083-00626ef2fc11&at=APZUnTWuXvVM2w1Q9h0hOsuX6thn:1709789680904",
"dest": "paper01.pdf"
"dest": "paper01_Gold.pdf"
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1UMimItX51tzNXkIMGPpHOdPNF5Dx0Tpy&export=download&authuser=0&confirm=t&uuid=97b668a6-2d0d-4389-ac5e-234e931b4328&at=APZUnTVvuvbAE8r7jpK8AkzGUzyw:1709790384938",
"dest": "ans.docx"
"dest": "ans_Gold.docx"
}
],
"result": [

4
evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json
View File

@@ -51,7 +51,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);"
]
}
}
@@ -60,7 +60,7 @@
"expected": {
"type": "rule",
"rules":{
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"]
"expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"]
}
},
"result": {

4
evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json
View File

@@ -1,7 +1,7 @@
{
"id": "81c425f5-78f3-4771-afd6-3d2973825947",
"snapshot": "libreoffice_calc",
"instruction": "Can you assist me in transferring the data from LibreOffice Calc in this file to a LibreOffice Writer table while preserving the original format as in calc file? Save the document as \"price.docx\" on the desktop.",
"instruction": "Can you assist me in transferring the data from LibreOffice Calc in the current sheet to a LibreOffice Writer table while preserving the original format as in calc file? Save the document as \"price.docx\" on the desktop.",
"source": "authors",
"config": [
{
@@ -63,7 +63,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1KbdlQC0qSAYewG8QnZgParnSwv3s3dub&export=download&authuser=0&confirm=t&uuid=15dcc25c-8168-425e-96e1-fd27e0d6904b&at=APZUnTVho4ZrREHf9DC4rKwdIi3R:1709557117932",
"dest": "price.docx"
"dest": "price_Gold.docx"
}
}
}

2
evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json
View File

@@ -56,7 +56,7 @@
"expected": {
"type": "rule",
"rules":{
"expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"]
"expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate"]
}
},
"result": {

2
evaluation_examples/examples/multi_apps/8df7e444-8e06-4f93-8a1a-c5c974269d82.json
View File

@@ -36,7 +36,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1IKRu-dMFP4Aqzq5-4TOmOWVci0qvC27K&export=download&authuser=0&confirm=t&uuid=e2dabad2-5648-4bc3-a40f-f008089cd613&at=APZUnTVh5JD5nT3EvutwHIaSnJAT:1709633945616",
"dest": "Recruitment_and_retention_of_health_professionals_across_Europe.zip"
"dest": "Recruitment_and_retention_of_health_professionals_across_Europe_Gold.zip"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json
View File

@@ -30,7 +30,7 @@
],
"trajectory": "trajectories/",
"related_apps": [
"os", "vlc"
"os", "vlc", "ubuntu_media_player"
],
"evaluator": {
"postconfig":[

2
evaluation_examples/examples/multi_apps/aad10cd7-9337-4b62-b704-a857848cedf2.json
View File

@@ -59,7 +59,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1r2KJv0H3foo1WlWnArxdXnaew-yixNqL&export=download&authuser=0&confirm=t&uuid=633cc27c-d38b-4c45-907d-025341b4af1c&at=APZUnTV8AW5F_aLVooprdfgt-Q-Z:1709547335200",
"dest": "notes.docx"
"dest": "notes_Gold.docx"
},
"result": {
"type": "vm_file",

2
evaluation_examples/examples/multi_apps/b337d106-053f-4d37-8da0-7f9c4043a66b.json
View File

@@ -1,7 +1,7 @@
{
"id": "b337d106-053f-4d37-8da0-7f9c4043a66b",
"snapshot": "os",
"instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display relative line numbers. Please search the internet for a tutorial on enabling relative line numbers and setting it as the default for my local Vim.",
"instruction": "Recently, I've been exploring the use of the Vim editor for code editing. However, the default settings don't display line numbers in Vim editor. Please search the internet for a tutorial on adding line numbers in Vim and setting it as default for my local Vim.",
"source": "authors",
"config": [
{

4
evaluation_examples/examples/multi_apps/bb83cab4-e5c7-42c7-a67b-e46068032b86.json
View File

@@ -54,7 +54,7 @@
}
}
],
"func": "compare_docx_tables",
"func": "compare_docx_files",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/script.docx",
@@ -63,7 +63,7 @@
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1-Sol1W8S7Ybj-3KBJJarbcYUqS5wAQ1C&export=download&authuser=0&confirm=t&uuid=d967f546-b8f7-4ac2-b8fc-b1635f1cfbc4&at=APZUnTUazmbS2X3BSXDEQtJgobgf:1709559012053",
"dest": "script.docx"
"dest": "script_Gold.docx"
}
}
}

93
evaluation_examples/examples/multi_apps/f5c13cdd-205c-4719-a562-348ae5cd1d91.json
View File

@@ -7,7 +7,11 @@
{
"type": "execute",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Departments/finance"]
"command": [
"mkdir",
"-p",
"/home/user/Documents/Departments/finance"
]
}
},
{
@@ -29,37 +33,52 @@
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xzv",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": ["nautilus", "/home/user/Documents/Departments/finance"]
"command": "/usr/bin/thunderbird -compose \"from='Anonym Tester <anonym-x2024@outlook.com>',subject='Reminder of Payment',body='$(cat /home/user/.payment-reminder-mail-body.html)'\"",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"nautilus",
"/home/user/Documents/Departments/finance"
]
}
}
],
"trajectory": "trajectories/f5c13cdd-205c-4719-a562-348ae5cd1d91",
"related_apps": ["thunderbird", "os", "libreoffice_calc"],
"related_apps": [
"thunderbird",
"os",
"libreoffice_calc"
],
"evaluator": {
"postconfig": [
{
"type": "sleep",
"parameters": {
"seconds": 10
}
}
],
"func": "check_accessibility_tree",
"result": {
"type": "accessibility_tree"
@@ -67,11 +86,27 @@
"expected": {
"type": "rule",
"rules": [
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"]},
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"]},
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"]},
{"selectors": ["tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"]}
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"fox@someuniversity.edu\"]"
]
},
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"iron@someuniversity.edu\"]"
]
},
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"nancy@someuniversity.edu\"]"
]
},
{
"selectors": [
"tool-bar[attr|id=MsgHeadersToolbar] label[name=To]~[attr|class=\"address-pill\"]>label[attr|class=\"pill-label\"][name*=\"stella@someuniversity.edu\"]"
]
}
]
}
}
}
}

2
evaluation_examples/examples/multi_apps/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -26,7 +26,7 @@
}
}
],
"trajectory": "trajectories/",
"trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef",
"related_apps": [
"vscode",
"os"