lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update multi apps

Browse Source
This commit is contained in:
tsuky_chen
2024-03-02 15:20:26 +08:00
parent 0f2ec5e692 9b7af944c4
commit 0554703c01
23 changed files with 801 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json Normal file
View File

@@ -0,0 +1,85 @@
{
"id": "00fa164e-2612-4439-992e-157d019a8436",
"snapshot": "libreoffice_writer",
"instruction": "I'm writing a report in Writer. A little data are recorded in \"~/Documents/awesome-desktop/expe-results.xlsx\". Help me to extract the results of LLM-based models and insert a table in the \"Main Results\" section of my report.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/awesome-desktop/"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx",
"url": "https://drive.google.com/uc?id=1oXro9_S7nZs_h_AZq5TJWEBofbrZM3oG&export=download"
},
{
"path": "/home/user/Documents/awesome-desktop/expe-results.xlsx",
"url": "https://drive.google.com/uc?id=1DwOikfnAumkZfvVcjIncgKZMITY3ruku&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx"
}
}
],
"trajectory": "trajectories/00fa164e-2612-4439-992e-157d019a8436",
"related_apps": [
"libreoffice_writer",
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "awe_desk_env.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1.0
}
}
],
"func": "compare_docx_tables",
"result": {
"type": "vm_file",
"path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx",
"dest": "awe_desk_env.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1hMZKSF74qBUvBDAu_zF8vHm6MKu96VBN&export=download",
"dest": "awe_desk_env_gt.docx"
}
}
}

45
evaluation_examples/examples/multi_apps/0c825995-5b70-4526-b663-113f4c999dd2.json Normal file
View File

@@ -0,0 +1,45 @@
{
"id": "0c825995-5b70-4526-b663-113f4c999dd2",
"snapshot": "libreoffice_calc",
"instruction": "I'm working on a comprehensive report for our environmental policy review meeting next week. I need to integrate key insights from an important document, which is a guidebook on the Green Economy, where I'm particularly interested in the 'Introduction' section. Could you extract this section and compile them into a new Google Doc named 'environment_policy_report (draft)'? This will significantly aid in our discussion on aligning our environmental policies with sustainable and green economic practices. Thanks!",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Desktop/wwf"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Desktop/wwf/lpr_living_planet_report_2016.pdf", "url": "https://drive.google.com/uc?id=19NCdw_MVP6nH5nC6okYYe8U1mJABfTRK&export=download"},
{"path": "/home/user/Desktop/wwf/279c656a32_ENGLISH_FULL.pdf", "url": "https://drive.google.com/uc?id=1ckH1NetfImQ9EyONTO-ZFWA8m8VIUFvD&export=download"},
{"path": "/home/user/Desktop/wwf/7g37j96psg_WWF_AR2021_spreads.pdf", "url": "https://drive.google.com/uc?id=1cxLTzmqDKMomOyvho29lvFvhRnb0Y8__&export=download"},
{"path": "/home/user/Desktop/GE Guidebook.pdf", "url": "https://drive.google.com/uc?id=1KzC_R3eI3Rmgwz5bkcI8Ohv7ebOrU-Is&export=download"},
{"path": "/home/user/Desktop/assessing_and_reporting_water_quality(q&a).pdf", "url": "https://drive.google.com/uc?id=1LFojf3Weflv3fVdrZrgTY1iUaRdbT9kG&export=download"}
]
}
}
],
"trajectory": "trajectories/0c825995-5b70-4526-b663-113f4c999dd2",
"related_apps": [
"libreoffice_calc",
"chrome",
"os"
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
},
"expected": {
},
"options": {
}
}
}

63
evaluation_examples/examples/multi_apps/26150609-0da3-4a7d-8868-0faf9c5f01bb.json Normal file
View File

@@ -0,0 +1,63 @@
{
"id": "26150609-0da3-4a7d-8868-0faf9c5f01bb",
"snapshot": "libreoffice_calc",
"instruction": "So, I've been dabbling with coding a Snake game in Python, and I finally got it up and running. It's pretty cool, but it's not without its quirks. The biggest issue I'm facing right now is that the snake can't seem to eat the food, no matter what. Could you help me tweak the code so the snake can actually eat the food? Thanks a bunch!",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/Desktop/snake"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Desktop/snake/food.py",
"url": "https://drive.usercontent.google.com/download?id=1Xkza2CaWyMPwnWiDmlP2lh1mvp6LuIJp&export=download&authuser=0&confirm=t&uuid=4e57edc7-5a7e-4a31-8436-656f3b398804&at=APZUnTWyVtrAaUST2_wI2GSeQrgV:1709303403572"
},
{
"path": "/home/user/Desktop/snake/main.py",
"url": "https://drive.usercontent.google.com/download?id=1jcYCRW9quS2aoO4MNbF6UFvca6Z6jO3P&export=download&authuser=0&confirm=t&uuid=a98c6bdc-4acb-43fe-864a-3461276c5ebb&at=APZUnTWQTXYHkJW9XiNgew5Qe3PI:1709303406766"
},
{
"path": "/home/user/Desktop/snake/settings.py",
"url": "https://drive.usercontent.google.com/download?id=1aGis8eNKqLM2ZFMIe0sgzaGA-IfTdM_H&export=download&authuser=0&confirm=t&uuid=178f1635-9538-467a-b191-1ecb8e76bc78&at=APZUnTVouOz4zz4wEaYp06nf31_4:1709303409591"
},
{
"path": "/home/user/Desktop/snake/snake.py",
"url": "https://drive.usercontent.google.com/download?id=1-EnaUyWeWmioOrd_S72_JN6xk5qdvJiK&export=download&authuser=0&confirm=t&uuid=6619d298-15a4-42f5-b3df-36f849e17aad&at=APZUnTXT-ko0Bc0uWau_tH3vkFnT:1709303412395"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"code",
"/home/user/Desktop/snake"
]
}
}
],
"trajectory": "trajectories/26150609-0da3-4a7d-8868-0faf9c5f01bb",
"related_apps": [
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
},
"expected": {
},
"options": {
}
}
}

18
evaluation_examples/examples/multi_apps/6d72aad6-187a-4392-a4c4-ed87269c51cf.json Normal file
View File

@@ -0,0 +1,18 @@
{
"id": "6d72aad6-187a-4392-a4c4-ed87269c51cf",
"snapshot": "libreoffice_calc",
"instruction": "Could you please converting OpenOffice Impress presentation to video and play it with VLC?",
"source": "https://superuser.com/questions/923171/converting-openoffice-impress-presentation-to-video-without-screen-recording",
"config": [
],
"trajectory": "trajectories/6d72aad6-187a-4392-a4c4-ed87269c51cf",
"related_apps": [
"libreoffice_calc",
"libreoffice_impress",
"libreoffice_writer",
"vlc"
],
"evaluator": {
"func": "infeasible"
}
}

107
evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json Normal file
View File

@@ -0,0 +1,107 @@
{
"id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"snapshot": "chrome",
"instruction": "Help me to get an initial setup of web extension with help of https://webext.eu . Tag the extension as \"happy-extension v0.0.1\". Leave description as blank for now. A background script and browser action is needed, while other features are not. Place the auto-generated folder under folder \"~/Projects\".",
"source": "authors",
"config": [],
"trajectory": "trajectories/74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"related_apps": [
"chrome",
"os"
],
"evaluator": {
"func": [
"check_json",
"diff_text_file",
"diff_text_file",
"diff_text_file",
"diff_text_file"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/manifest.json",
"dest": "manifest.json"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/background_script.js",
"dest": "background_script.js"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/index.html",
"dest": "index.html"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/style.css",
"dest": "style.css"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/script.js",
"dest": "script.js"
}
],
"expected": [
{
"type": "rule",
"rules": {
"expect": [
{
"key": ["name"],
"method": "eq",
"ref": "happy-extension"
},
{
"key": ["version"],
"method": "eq",
"ref": "0.0.1"
},
{
"key": ["background", "scripts"],
"method": "eq",
"ref": ["background_script.js"]
},
{
"key": ["browser_action", "default_icon"],
"method": "eq",
"ref": {"64": "icons/icon.png"}
},
{
"key": ["browser_action", "default_popup"],
"method": "eq",
"ref": "browserAction/index.html"
},
{
"key": ["browser_action", "default_title"],
"method": "eq",
"ref": "happy-extension"
}
]
}
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1t5Llhn6seDUXVs-eILu6CjwFEQL9Z5Qm&export=download",
"dest": "background_script.js"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=19fMAsWd6q4ElLdOceJ-otHbxRJA_pc_U&export=download",
"dest": "index.html"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1fwfiRPjdug8uh6z23RFO1JtlGH_L_Hl_&export=download",
"dest": "style.css"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=14YYnhCfRtHQNk8M4fBPaUQeteoFMGBsA&export=download",
"dest": "script.js"
}
]
}
}

27
evaluation_examples/examples/multi_apps/767a3271-56db-4745-ac5d-846ef05e6fe5.json Normal file
View File

@@ -0,0 +1,27 @@
{
"id": "767a3271-56db-4745-ac5d-846ef05e6fe5",
"snapshot": "libreoffice_calc",
"instruction": "Hey there! I've been swamped with emails lately, and I'm trying to get organized. I'm part of a local community group, and we've been receiving a lot of emails about different events and volunteer opportunities. I need to sort through these emails and pull out specific information to keep track of everything. Could you help me extract details from emails that mention 'volunteer opportunities' and organize them into a spreadsheet? I'm looking for the event name, date, location, and contact person's email. It would be a huge help if we could have this info neatly laid out so we can easily see what's coming up and who to contact. Thanks a bunch!",
"source": "authors",
"config": [
],
"trajectory": "trajectories/767a3271-56db-4745-ac5d-846ef05e6fe5",
"related_apps": [
"thunderbird",
"libreoffice_calc"
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
},
"expected": {
},
"options": {
}
}
}

96
evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json Normal file
View File

@@ -0,0 +1,96 @@
{
"id": "7e287123-70ca-47b9-8521-47db09b69b14",
"snapshot": "libreoffice_calc",
"instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for 2019~2023 in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Fundings/ecs", "/home/user/Documents/Fundings/grf"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Fundings/ecs/ecs15.pdf", "url": "https://drive.google.com/uc?id=1FTiT3mLlkehe2yWVdSMWr1w4ltLtQZUy&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs16.pdf", "url": "https://drive.google.com/uc?id=1DMzZyhDey3lDsQ7fcPiJm9AYGECZms3q&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs17.pdf", "url": "https://drive.google.com/uc?id=1TgAWk7FiV8fNrG2L3-Eu7BDccqtXebXY&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs23.pdf", "url": "https://drive.google.com/uc?id=11DVxH4eRjECUxZNbUhYNJhgD0Y5WoN8r&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs22.pdf", "url": "https://drive.google.com/uc?id=17IgyJADA65F40kH79S90QgEzPa7IERXx&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs21.pdf", "url": "https://drive.google.com/uc?id=1kB4eFfLisPXKOirGUHbbcOyf73t7MVqL&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs20.pdf", "url": "https://drive.google.com/uc?id=179j9tD1xRSgd9COM7rzErO6FLYO2sc_a&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs19.pdf", "url": "https://drive.google.com/uc?id=1ogZUTfKjvQhg58GXtVUxe1U8VvHU-3ap&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs18.pdf", "url": "https://drive.google.com/uc?id=1MfuvRhAnhMEMbxn5js2ffWqMHJx2fsd2&export=download"},
{"path": "/home/user/Documents/Fundings/grf/customer-information-sheet-for-inward-payments-to-hong-kong.pdf", "url": "https://drive.google.com/uc?id=1s-H3an7HLBM9ku6d6Hcdj1qkSwKAHngU&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf15.pdf", "url": "https://drive.google.com/uc?id=1rRQXo9XHnCVTG8XqNAv0SJwPTW36MMbm&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf16.pdf", "url": "https://drive.google.com/uc?id=18ljRqkdyXEZ464E0dpKjaEa2NFexyw3I&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf17.pdf", "url": "https://drive.google.com/uc?id=1VrqOnyhpOkMpyIJ6YMrAhixpahjYonOd&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf18.pdf", "url": "https://drive.google.com/uc?id=182CLDUr372-jpAiY4YvSbGNXF9TsWxzA&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf19.pdf", "url": "https://drive.google.com/uc?id=1YkJtjlklKN0NmLiI2Hi4f_dKtTm5SPxT&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf20.pdf", "url": "https://drive.google.com/uc?id=1a7Uc7VCMlEX6fy-5oqE6i1YLitBe7gaf&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf21.pdf", "url": "https://drive.google.com/uc?id=1s8km4Wle4lc5PkbUQfivBFK0IJQgxMiB&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf22.pdf", "url": "https://drive.google.com/uc?id=1HTEb1uK7LNvbVyeXgO8WemCPPgiASKiy&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf23.pdf", "url": "https://drive.google.com/uc?id=1XM-jZlfu_i4waDZHb8Z6Vr5b3LgULTtP&export=download"}
]
}
}
],
"trajectory": "trajectories/7e287123-70ca-47b9-8521-47db09b69b14",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Desktop",
"/home/user/Desktop/GRF-p5y.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/GRF-p5y.xlsx",
"/home/user/Desktop/GRF-p5y-Sheet1.csv"
],
"dest": [
"GRF-p5y.xlsx",
"GRF-p5y-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1fDM4Y-WuFCnfksPLgynj-WSmzbqn2TcV&export=download",
"https://drive.google.com/uc?id=1waThupubGOJop0FU0b0yhT6QnjNYkLGy&export=download"
],
"dest": [
"GRF-p5y-gt.xlsx",
"GRF-p5y-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}

124
evaluation_examples/examples/multi_apps/869de13e-bef9-4b91-ba51-f6708c40b096.json
View File

@@ -1,8 +1,8 @@
{
"id": "869de13e-bef9-4b91-ba51-f6708c40b096",
"snapshot": "os",
"snapshot": "libreoffice_calc",
"instruction": "Can you organize my desktop by identifying schoolwork-related files, coding projects, and random downloads, ensuring no file is misplaced? Specifically, schoolwork files should end up in the 'Work' folder, coding projects in 'Projects', and everything else in 'Miscellaneous'. Use content analysis for files without clear extensions or names.",
"source": "",
"source": "authors",
"config": [
{
"type": "command",
@@ -39,46 +39,114 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052",
"path": "/home/user/project/main.py"
"url": "https://drive.usercontent.google.com/download?id=1DYDA4Sig4--o4aRV8FY72qzhCyPGHHzF&export=download&authuser=0&confirm=t&uuid=5a78b6b5-590a-47ab-85a8-0381b8d3f18e&at=APZUnTXch_5fCuVoME_uNsgMENE3:1709360319470",
"path": "/home/user/Desktop/2-if-for-array.zip"
},
{
"url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290",
"path": "/home/user/project/README.md"
"url": "https://drive.usercontent.google.com/download?id=1J7ySYpMJ4oSJg3f8KxQjLvKbq34uwxdI&export=download&authuser=0&confirm=t&uuid=e3b0d7a0-aea8-4e87-9b34-a798e0184c43&at=APZUnTVIfKSLCAC9k_9JveYeKHmO:1709357169335",
"path": "/home/user/Desktop/07-cluster-kMean (1).ppt"
},
{
"url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443",
"path": "/home/user/project/.vscode/settings.json"
"url": "https://drive.usercontent.google.com/download?id=1QB7uREIPXtB6Gshrpm0sQs7aUDuWF2NF&export=download&authuser=0&confirm=t&uuid=2f77bf2d-d5f8-47a6-b631-050595be0973&at=APZUnTUKtDOGJbejBN_A-jWeex4W:1709357171790",
"path": "/home/user/Desktop/2023_validation_7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1UaDMWrkkWQvpf2yjyz_MQdwNj8hfVSTD&export=download&authuser=0&confirm=t&uuid=0239206d-d5ad-4528-82d4-ee1b08205aca&at=APZUnTVYa7dNgIQToGhdeSKV9gnN:1709357271165",
"path": "/home/user/Desktop/assign1-data3.zip"
},
{
"url": "https://drive.usercontent.google.com/download?id=1XCYElfSGeyLzB-xdNE6BB7FpZlGVpEKU&export=download&authuser=0&confirm=t&uuid=ae66fe54-16ed-4e8e-a90c-8b3f3cea39d1&at=APZUnTU82gfcZaXvwEIYfd3kLMSW:1709357279299",
"path": "/home/user/Desktop/assignment_mark_frontpage.docx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1EMBV2giW40-EM7IU3Y_QApus252sOvsP&export=download&authuser=0&confirm=t&uuid=53228262-9385-4ef0-bb81-dd000be41c8e&at=APZUnTVCd7qkZwerJGRy-_qL9Cb_:1709357381926",
"path": "/home/user/Desktop/cco-return-to-school-survey-underlying-data-tables.xlsx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1AGtaH_HDC70qcwgZsgEYu_1DZ7Rw8s9P&export=download&authuser=0&confirm=t&uuid=e44c4ab0-69ce-4b5e-9eb6-9e5eeda3c06c&at=APZUnTXre_L2EBLsjjKIKBlNEtpj:1709357396112",
"path": "/home/user/Desktop/DOC_2480903712718068684.pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=1Z_9uRyDLpCrkAJOmnwRwoEz2NxeTT545&export=download&authuser=0&confirm=t&uuid=4ea4b2a6-6e71-4de3-907e-94d77aeef8aa&at=APZUnTUzD9261CWKMX6NC0sItXps:1709357398907",
"path": "/home/user/Desktop/Family Status Equality-Eng (Aug 2021).pdf"
},
{
"url": "https://drive.usercontent.google.com/download?id=110FbG5m_XDDUpO3bvUZTPzw9y0ce1KB5&export=download&authuser=0&confirm=t&uuid=cbce5e69-d4dd-4508-8192-647d6da54fc0&at=APZUnTWxxAS53KTZJklxT6riooU6:1709357558943",
"path": "/home/user/Desktop/IA_Format.docx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1O1yd3Dm_nKEtknQYzDAHqPllv0BeQ2bp&export=download&authuser=0&confirm=t&uuid=636ee37e-4ee1-4415-ba71-73e42335c4bf&at=APZUnTW7tJhD6LG7CglaqI_y0Exu:1709357564381",
"path": "/home/user/Desktop/S1578219008703378.pdf"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"cd /home/user/Desktop && unzip -q assign1-data3.zip && rm -rf assign1-data3.zip && unzip -q 2-if-for-array.zip && rm -rf 2-if-for-array.zip && rm -rf __MACOSX"
]
}
}
],
"trajectory": "trajectories/",
"trajectory": "trajectories/869de13e-bef9-4b91-ba51-f6708c40b096",
"related_apps": [
"os",
"libreoffice_calc",
"libreoffice_writer",
"libreoffice_calc",
"os",
"libreoffice_impress"
],
"evaluator": {
"func": "is_extension_installed",
"result": {
"type": "vm_command_line",
"command": [
"ls",
"/home/user/project/",
"|",
"grep",
"project.code-workspace"
]
},
"expected": {
"type": "rule",
"rules": {
"type": "contain",
"expected": "project.code-workspace"
"func": [
"exact_match",
"exact_match",
"exact_match"
],
"result": [
{
"type": "vm_command_line",
"command": [
"ls",
"/home/user/Desktop/Work"
]
},
{
"type": "vm_command_line",
"command": [
"ls",
"/home/user/Desktop/Projects"
]
},
{
"type": "vm_command_line",
"command": [
"ls",
"/home/user/Desktop/Miscellaneous"
]
}
}
],
"expected": [
{
"type": "rule",
"rules": {
"expected": "07-cluster-kMean (1).ppt\nassignment_mark_frontpage.docx\ncco-return-to-school-survey-underlying-data-tables.xlsx\nDOC_2480903712718068684.pdf\nIA_Format.docx\n"
}
},
{
"type": "rule",
"rules": {
"expected": "2-if-for-array\nassign1-data_python3\n"
}
},
{
"type": "rule",
"rules": {
"expected": "2023_validation_7bd855d8-463d-4ed5-93ca-5fe35145f733.xlsx\nFamily Status Equality-Eng (Aug 2021).pdf\nS1578219008703378.pdf\n"
}
}
]
}
}

67
evaluation_examples/examples/multi_apps/9219480b-3aed-47fc-8bac-d2cffc5849f7.json Normal file
View File

@@ -0,0 +1,67 @@
{
"id": "9219480b-3aed-47fc-8bac-d2cffc5849f7",
"snapshot": "libreoffice_calc",
"instruction": "Hi, I recently playing with developing a small python-based teris game. While I have finished most of the part, something is wrong under some cases when I press up to rotate, the whole program will crash, please run the code for me and fix the bugs of code.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": [
"mkdir",
"-p",
"/home/user/Desktop/teris"
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Desktop/teris/block.py",
"url": "https://drive.usercontent.google.com/download?id=1txPwaWML0e8cjpDW-nw5N6HOC6fYwfI1&export=download&authuser=0&confirm=t&uuid=0b69795a-3600-4ec4-be9b-533deeb76e94&at=APZUnTVK-VGsZfTufLSh_3eRigYb:1709303077428"
},
{
"path": "/home/user/Desktop/teris/main.py",
"url": "https://drive.usercontent.google.com/download?id=1vJ2FEw7RNfRr0KBynHM36_nNAc4jTArl&export=download&authuser=0&confirm=t&uuid=11f697bc-1414-46a0-bc2d-b2c49686d85e&at=APZUnTXCtKDpqTAOstIioRrJzx67:1709303181059"
},
{
"path": "/home/user/Desktop/teris/settings.py",
"url": "https://drive.usercontent.google.com/download?id=1tYI8QZz-T-sNFRAMaYnzFv80upIveUT-&export=download&authuser=0&confirm=t&uuid=50c64eed-ab7e-4dbe-82ce-d01b25364556&at=APZUnTXUok8Cos6rUo-fnKLwnPmn:1709303184763"
},
{
"path": "/home/user/Desktop/teris/teris.py",
"url": "https://drive.usercontent.google.com/download?id=1UOJp5Y6JLmaXmZXHc_GkM3nMfKbS6Dyu&export=download&authuser=0&confirm=t&uuid=3189e5d6-5984-45c7-9dd8-6f8ec3a0845e&at=APZUnTWYTtsu27Ds3aHrZpIAnNcA:1709303187967"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"code",
"/home/user/Desktop/teris"
]
}
}
],
"trajectory": "trajectories/9219480b-3aed-47fc-8bac-d2cffc5849f7",
"related_apps": [
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
},
"expected": {
},
"options": {
}
}
}

94
evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json Normal file
View File

@@ -0,0 +1,94 @@
{
"id": "b5062e3e-641c-4e3a-907b-ac864d2e7652",
"snapshot": "libreoffice_calc",
"instruction": "I've got a mass of different categories of paper in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Papers"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Documents/Papers/zhang_appagent.pdf",
"url": "https://arxiv.org/pdf/2312.13771.pdf"
},
{
"path": "/home/user/Documents/Papers/niu_screenagent.pdf",
"url": "https://arxiv.org/pdf/2402.07945.pdf"
},
{
"path": "/home/user/Documents/Papers/koh_visualwebarena.pdf",
"url": "https://arxiv.org/pdf/2401.13649.pdf"
},
{
"path": "/home/user/Documents/Papers/deng_mind2web.pdf",
"url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf"
}
]
}
}
],
"trajectory": "trajectories/b5062e3e-641c-4e3a-907b-ac864d2e7652",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user",
"/home/user/authors.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/authors.xlsx",
"/home/user/authors-Sheet1.csv"
],
"dest": [
"authors.xlsx",
"authors-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1fttbvfHuoQfsQUk3fVXkJsCu231jhnQj&export=download",
"https://drive.google.com/uc?id=1fq4hbk1g9R_SjknzwFAqvyF1ICyNYfok&export=download"
],
"dest": [
"authors-gt.xlsx",
"authors-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"ignore_case": true
}
]
}
}
}

64
evaluation_examples/examples/multi_apps/d1acdb87-bb67-4f30-84aa-990e56a09c92.json Normal file
View File

@@ -0,0 +1,64 @@
{
"id": "d1acdb87-bb67-4f30-84aa-990e56a09c92",
"snapshot": "libreoffice_calc",
"instruction": "Can you help me find these restaurants on Google Maps? I have a list of places I've been wanting to visit, and I need to gather some detailed information about them, including the address, what people have to say about them (ratings), and the hours of operation. It would be great if you could search for each of them on Google Maps and help me jot down this information into my sheet file, thanks!",
"source": "authors",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Desktop/restaurants.txt",
"url": "https://drive.google.com/uc?id=1IehFLJPZcFv8Ujk31ExbyGLji9AylmmJ&export=download"
},
{
"path": "/home/user/Desktop/MUST_VISIT.xlsx",
"url": "https://drive.google.com/uc?id=1fXmjvZcwkIcckMIAXi3Hv_JAbVWpgs_l&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/MUST_VISIT.xlsx"
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/restaurants.txt"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 5
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "restaurants.txt (~/Desktop) - gedit",
"strict": true
}
}
],
"trajectory": "trajectories/d1acdb87-bb67-4f30-84aa-990e56a09c92",
"related_apps": [
"os",
"chrome",
"libreoffice_calc"
],
"evaluator": {
"postconfig": [],
"func": "",
"result": {
},
"expected": {
},
"options": {
}
}
}