lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finish multi-app examples

Browse Source
This commit is contained in:
rhythmcao
2024-02-01 00:53:31 +08:00
parent c7fb35de50
commit fc15a33b70
13 changed files with 844 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
evaluation_examples/examples/multi_apps/78aed49a-a710-4321-a793-b611a7c5b56b.json Normal file
View File

@@ -0,0 +1,177 @@
{
"id": "78aed49a-a710-4321-a793-b611a7c5b56b",
"snapshot": "chrome",
"instruction": "Could you help me save all attachments of the oldest email in Bills local folders to the attachment/ folder in Google Drive and then move this email to a different folder \"have_seen\" in Local Folders.",
"source": "https://marketplace.uipath.com/listings/upload-email-attachments-from-gmail-to-google-drive",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'attachment' and 'root' in parents and trashed = false",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://stackoverflow.com/",
"https://paperswithcode.com/"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1W18fZ4Dog-cSH3Fxa3bcfTz28z6AsL_1&export=download&authuser=0&confirm=t&uuid=7c8fc164-6e7b-4849-aba1-f3dba947feba&at=APZUnTWd2DFbIYnlMf-GY4qMqXcc:1706682437093",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": [
"check_thunderbird_folder",
"compare_pdfs"
],
"result": [
{
"type": "vm_file",
"multi": true,
"path": [
"/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders/have_seen",
"/home/user/.thunderbird/t5q2a5hp.default-release/Mail/Local Folders/Bills"
],
"dest": [
"have_seen",
"Bills"
],
"gives": [
0,
1
]
},
{
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"path_list": [
[
"attachment",
"Invoice-01234567-89ab.pdf"
],
[
"attachment",
"Receipt-0123-4567.pdf"
]
],
"dest": [
"Invoice-01234567-89ab.pdf",
"Receipt-0123-4567.pdf"
]
}
],
"expected": [
{
"type": "cloud_file",
"multi": true,
"path": [
"https://drive.usercontent.google.com/download?id=12ielR0p3jFZ4UDfGlC7fdF88uUTM_3bn&export=download&authuser=0&confirm=t&uuid=6ef47d29-2772-49cd-87e7-17efd587f1ba&at=APZUnTUGjXxDcDcMF5RoAsVtbFOB:1706689562530",
"https://drive.usercontent.google.com/download?id=13XQ4Hr62WyHVwpefjD-tgkpjb0p9HKpe&export=download&authuser=0&confirm=t&uuid=ad4ee033-83e4-4df4-9f9c-ac61d43634bc&at=APZUnTV5ZI1GFXNHTyIblwSKjyku:1706692277803"
],
"dest": [
"have_seen_gold",
"Bills_gold"
],
"gives": [
0,
1
]
},
{
"type": "cloud_file",
"multi": true,
"path": [
"https://drive.usercontent.google.com/download?id=1SlzOPRfzaKyhBERy-ks2-rjzrjtvgoDG&export=download&authuser=0&confirm=t&uuid=e4cb717b-fb5d-4860-99d9-9e1f38df2592&at=APZUnTXlcSimcf9qZ7uZmlAUj-zQ:1706683984205",
"https://drive.usercontent.google.com/download?id=19SdAYymlHvFQ7wzc-_JfFYOYMW6xAw5-&export=download&authuser=0&confirm=t&uuid=e1cbff64-d615-493d-9d02-bcdbd40f878c&at=APZUnTVRpcei6q_BjzQ7EtHA7voq:1706683990388"
],
"dest": [
"Invoice-01234567-89ab_gold.pdf",
"Receipt-0123-4567_gold.pdf"
],
"gives": [
0,
1
]
}
],
"options": [
{
"remove_deleted": true,
"remove_duplicate": true
},
{}
]
}
}

132
evaluation_examples/examples/multi_apps/a0b9dc9c-fc07-4a88-8c5d-5e3ecad91bcb.json Normal file
View File

@@ -0,0 +1,132 @@
{
"id": "a0b9dc9c-fc07-4a88-8c5d-5e3ecad91bcb",
"snapshot": "chrome",
"instruction": "Please help me backup my emails in \"Bills\" folder in Thunderbird and store the .eml files with only subject names to my Google Drive folder called \"emails\".",
"source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive",
"config": [
{
"type": "googledrive",
"parameters": {
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"operation": ["delete"],
"args": [
{
"query": "title = 'emails' and 'root' in parents",
"trash": false
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://arxiv.org/",
"https://github.com/ohmyzsh/ohmyzsh/wiki/themes",
"https://releases.ubuntu.com/"
]
}
},
{
"type": "login",
"parameters": {
"settings_file": "evaluation_examples/settings/google/settings.json",
"platform": "googledrive"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1cATYRGGbZ_vZEjJWajI1Dn4gmp3Won-l&export=download&authuser=0&confirm=t&uuid=156022ae-a56a-400c-a934-34e4369ec82a&at=APZUnTWMnGMhJciITn7IvRY33zuJ:1706707804986",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": "check_thunderbird_folder",
"result": {
"type": "googledrive_file",
"settings_file": "evaluation_examples/settings/googledrive/settings.yml",
"path_list": [
[
"emails",
"Amazon Web Services Invoice Available [Account: 0123456789ab] [Invoice ID: 0123456789].eml"
],
[
"emails",
"Your receipt from X (formerly Twitter) #0123-4567.eml"
]
],
"dest": [
"pred1.eml",
"pred2.eml"
]
},
"expected": {
"type": "cloud_file",
"multi": "true",
"path": [
"https://drive.usercontent.google.com/download?id=1Fb2ofAjfn-wlIYbtIGXNvLJEB85KbO7Y&export=download&authuser=0&confirm=t&uuid=9a996347-9093-43ed-8ad2-7e769e0576bd&at=APZUnTUp2pvXjM0zGvjc22lewOv3:1706708252395",
"https://drive.usercontent.google.com/download?id=1LZ2PgwmVgO62dNOueWy7RLlJ1_d8Hz3E&export=download&authuser=0&confirm=t&uuid=f632abc2-3ff7-4e9f-a7c6-72c9bbc44654&at=APZUnTXihve7i15GwLyEx2rwDFUk:1706708265408"
],
"dest": [
"gold1.eml",
"gold2.eml"
],
"gives": [
0,
1
]
}
}
}

98
evaluation_examples/examples/multi_apps/c867c42d-a52d-4a24-8ae3-f75d256b5618.json Normal file
View File

@@ -0,0 +1,98 @@
{
"id": "c867c42d-a52d-4a24-8ae3-f75d256b5618",
"snapshot": "thunderbird",
"instruction": "Please assist me in exporting my contacts of Personal Address Book from Thunderbird into contacts.csv file in the desktop and convert it to .xlsx with Libreoffice Calc.",
"source": "https://www.sync.blue/en/sync/mozilla-thunderbird/google-sheets/",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"libreoffice",
"--calc"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1wKXmJ14dnxSzdy9ZF_ePWU7zpevY6Dry&export=download&authuser=0&confirm=t&uuid=9b476c95-8eee-4a9a-8cee-c3620d5ce250&at=APZUnTUzDeeeMNr34DB1vEnBK6N7:1706719624132",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"--recursive-unlink",
"-xz",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"libreoffice_calc"
],
"evaluator": {
"func": [
"compare_csv",
"compare_table"
],
"conj": "and",
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/contacts.csv",
"dest": "contacts.csv"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/contacts.xlsx",
"dest": "contacts.xlsx"
}
],
"expected": [
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1StwASpAR2ALq2Y1vugGsdUJptg6FwjEm&export=download&authuser=0&confirm=t&uuid=56339e19-b889-4da1-ab72-5e0b90f13fff&at=APZUnTVWFF2pBrtWU_hXgzfbrWP2:1706719668676",
"dest": "contacts_gold.csv"
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1s25eUpvkMzSm6p_WA7O13t6mVqmkxr2C&export=download&authuser=0&confirm=t&uuid=901cbd32-6026-4391-a5cc-989e1047cf7c&at=APZUnTUs27mZceDshB_f9Tx4PFyz:1706719610831",
"dest": "contacts_gold.xlsx"
}
],
"options": [
{},
{
"rules": [
{
"type": "sheet_data",
"sheet_idx0": "RI0",
"sheet_idx1": "EI0"
}
]
}
]
}
}

68
evaluation_examples/examples/multi_apps/d9b7c649-c975-4f53-88f5-940b29c47247.json Normal file
View File

@@ -0,0 +1,68 @@
{
"id": "d9b7c649-c975-4f53-88f5-940b29c47247",
"snapshot": "thunderbird",
"instruction": "Help me extract the latest 5 emails in daily folder from Thunderbird, from the earliest to the most recent by time, and creates a LibreOffice Calc Report \"report.xlsx\" in the desktop, storing the sender_name, sender_address, subject, CC, and number_of_attachments.",
"source": "https://marketplace.uipath.com/listings/extract-the-first-1000-gmail-emails-from-the-current-month-in-a-new-google-sheets-report",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1aCmZvSIs8Mb2kM_IVyhTbLawFvHOrlW-&export=download&authuser=0&confirm=t&uuid=dee0fe8b-731b-4bb7-97c4-02a2d8154da8&at=APZUnTVO-WjksU7WYUq4sCkNMlsL:1706710969972",
"path": "/home/user/thunderbird-profile.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"libreoffice_calc"
],
"evaluator": {
"func": "compare_table",
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/report.xlsx",
"dest": "report.xlsx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=13L73ukCzuLYUgY11xz2b6DEoYhFvmXVE&export=download&authuser=0&confirm=t&uuid=82e92b1f-d4c0-44e6-b40f-595dff880acd&at=APZUnTXzPvN60uTLjy7QSPEF2Ft9:1706714031096",
"dest": "report_gold.xlsx"
},
"options": {
"rules": [
{
"type": "sheet_data",
"sheet_idx0": "RI0",
"sheet_idx1": "EI0"
}
]
}
}
}

91
evaluation_examples/examples/multi_apps/e135df7c-7687-4ac0-a5f0-76b74438b53e.json Normal file
View File

@@ -0,0 +1,91 @@
{
"id": "e135df7c-7687-4ac0-a5f0-76b74438b53e",
"snapshot": "libreoffice_calc",
"instruction": "Please convert a .xlsx file opened in LibreOffice Calc to a .html file and view it in Chrome.",
"source": "https://www.ilovefreesoftware.com/23/featured/free-csv-to-html-converter-software-windows.html",
"config": [
{
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://aclanthology.org/",
"https://openai.com/",
"https://www.linkedin.com/home/"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1zio_nULUdQGFGFBD55aZ52vhqYBtlQeJ&export=download&authuser=0&confirm=t&uuid=68483de0-2035-461d-90d5-e4048825d1ce&at=APZUnTXRMiB4UDzbsqrgm6BbFefE:1706630941497",
"path": "/home/user/Desktop/annual-enterprise-survey-2021-financial-year-provisional.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/annual-enterprise-survey-2021-financial-year-provisional.xlsx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"thunderbird",
"chrome"
],
"evaluator": {
"func": ["is_expected_tabs", "compare_htmls"],
"result": [
{
"type": "open_tabs_info"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/annual-enterprise-survey-2021-financial-year-provisional.html",
"dest": "annual-enterprise-survey-2021-financial-year-provisional.html"
}
],
"expected": [
{
"type": "rule",
"rules": {
"type": "url",
"urls": [
"https://aclanthology.org/",
"https://openai.com/",
"https://www.linkedin.com/home/",
"file:///home/user/Desktop/annual-enterprise-survey-2021-financial-year-provisional.html"
]
}
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1HvdTjLZctQGgo3BojmPBiSrPXFkqubJi&export=download&authuser=0&confirm=t&uuid=47f78e2c-bc38-416b-900f-4837ff588bfa&at=APZUnTXSartmo7MSTsanrkc9zHpm:1706680670213",
"dest": "annual-enterprise-survey-2021-financial-year-provisional_gold.html"
}
]
}
}

121
evaluation_examples/examples/multi_apps/f7dfbef3-7697-431c-883a-db8583a4e4f9.json Normal file
View File

@@ -0,0 +1,121 @@
{
"id": "f7dfbef3-7697-431c-883a-db8583a4e4f9",
"snapshot": "libreoffice_writer",
"instruction": "Could you convert all `.doc` files in current directory to PDF all at once in the command line?",
"source": "https://www.thegeekdiary.com/libreoffice-command-examples-in-linux/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1efUzatD7vixloPY9-XENOL3OZ5__1hod&export=download&authuser=0&confirm=t&uuid=14a3a6b1-a6e2-46cf-9e5b-bcc9e2acbc98&at=APZUnTWtyQUzT6eO5ZTcnmAZkwyX:1706704733958",
"path": "/home/user/Desktop/doc.tar.gz"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"tar -zxf /home/user/Desktop/doc.tar.gz -C /home/user/Desktop/ && rm /home/user/Desktop/doc.tar.gz"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"history -c && echo > ~/.bash_history && sleep 3"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gnome-terminal",
"--maximize",
"--working-directory=/home/user/Desktop"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_writer",
"terminal"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"cd /home/user/Desktop && tar -zcf pdf.tar.gz *.pdf"
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"killall gnome-terminal-server"
]
}
}
],
"func": [
"check_include_exclude",
"compare_archive"
],
"result": [
{
"type": "vm_command_line",
"command": [
"/bin/bash",
"-c",
"output=$(cat ~/.bash_history | grep -E \"(soffice|libreoffice).+--convert-to\\s+pdf.+\\*\\.doc\"); if [ -z \"$output\" ]; then echo \"failed to complete this task\"; else echo \"catch the desired command\"; fi"
]
},
{
"type": "vm_file",
"path": "/home/user/Desktop/pdf.tar.gz",
"dest": "pdf.tar.gz"
}
],
"expected": [
{
"type": "rule",
"rules": {
"include": [
"catch the desired command"
],
"exclude": [
"failed to complete this task"
]
}
},
{
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1A-UoQdQvvtFBQWkPi_Q8ecNe4gjlh0dg&export=download&authuser=0&confirm=t&uuid=0c97e2f6-3de8-438f-a676-557cca5ee292&at=APZUnTUy4pghj5TtS-suVMXrCLO2:1706705033330",
"dest": "pdf_gold.tar.gz"
}
],
"options": [
{},
{
"file_type": "pdf"
}
]
}
}