lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-03-01 15:11:41 +08:00
parent f0a20cdbd1 58e21ab871
commit 96188217c1
6 changed files with 227 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
evaluation_examples/examples/multi_apps/74d5859f-ed66-4d3e-aa0e-93d7a592ce41.json Normal file
View File

@@ -0,0 +1,107 @@
{
"id": "74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"snapshot": "chrome",
"instruction": "Help me to get an initial setup of web extension with help of https://webext.eu . Tag the extension as \"happy-extension v0.0.1\". Leave description as blank for now. A background script and browser action is needed, while other features are not. Place the auto-generated folder under folder \"~/Projects\".",
"source": "authors",
"config": [],
"trajectory": "trajectories/74d5859f-ed66-4d3e-aa0e-93d7a592ce41",
"related_apps": [
"chrome",
"os"
],
"evaluator": {
"func": [
"check_json",
"diff_text_file",
"diff_text_file",
"diff_text_file",
"diff_text_file"
],
"result": [
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/manifest.json",
"dest": "manifest.json"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/background_script.js",
"dest": "background_script.js"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/index.html",
"dest": "index.html"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/style.css",
"dest": "style.css"
},
{
"type": "vm_file",
"path": "/home/user/Projects/happy-extension/browserAction/script.js",
"dest": "script.js"
}
],
"expected": [
{
"type": "rule",
"rules": {
"expect": [
{
"key": ["name"],
"method": "eq",
"ref": "happy-extension"
},
{
"key": ["version"],
"method": "eq",
"ref": "0.0.1"
},
{
"key": ["background", "scripts"],
"method": "eq",
"ref": ["background_script.js"]
},
{
"key": ["browser_action", "default_icon"],
"method": "eq",
"ref": {"64": "icons/icon.png"}
},
{
"key": ["browser_action", "default_popup"],
"method": "eq",
"ref": "browserAction/index.html"
},
{
"key": ["browser_action", "default_title"],
"method": "eq",
"ref": "happy-extension"
}
]
}
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1t5Llhn6seDUXVs-eILu6CjwFEQL9Z5Qm&export=download",
"dest": "background_script.js"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=19fMAsWd6q4ElLdOceJ-otHbxRJA_pc_U&export=download",
"dest": "index.html"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1fwfiRPjdug8uh6z23RFO1JtlGH_L_Hl_&export=download",
"dest": "style.css"
},
{
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=14YYnhCfRtHQNk8M4fBPaUQeteoFMGBsA&export=download",
"dest": "script.js"
}
]
}
}

96
evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json Normal file
View File

@@ -0,0 +1,96 @@
{
"id": "7e287123-70ca-47b9-8521-47db09b69b14",
"snapshot": "libreoffice_calc",
"instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for the past five years in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Fundings/ecs", "/home/user/Documents/Fundings/grf"]
}
},
{
"type": "download",
"parameters": {
"files": [
{"path": "/home/user/Documents/Fundings/ecs/ecs15.pdf", "url": "https://drive.google.com/uc?id=1FTiT3mLlkehe2yWVdSMWr1w4ltLtQZUy&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs16.pdf", "url": "https://drive.google.com/uc?id=1DMzZyhDey3lDsQ7fcPiJm9AYGECZms3q&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs17.pdf", "url": "https://drive.google.com/uc?id=1TgAWk7FiV8fNrG2L3-Eu7BDccqtXebXY&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs23.pdf", "url": "https://drive.google.com/uc?id=11DVxH4eRjECUxZNbUhYNJhgD0Y5WoN8r&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs22.pdf", "url": "https://drive.google.com/uc?id=17IgyJADA65F40kH79S90QgEzPa7IERXx&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs21.pdf", "url": "https://drive.google.com/uc?id=1kB4eFfLisPXKOirGUHbbcOyf73t7MVqL&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs20.pdf", "url": "https://drive.google.com/uc?id=179j9tD1xRSgd9COM7rzErO6FLYO2sc_a&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs19.pdf", "url": "https://drive.google.com/uc?id=1ogZUTfKjvQhg58GXtVUxe1U8VvHU-3ap&export=download"},
{"path": "/home/user/Documents/Fundings/ecs/ecs18.pdf", "url": "https://drive.google.com/uc?id=1MfuvRhAnhMEMbxn5js2ffWqMHJx2fsd2&export=download"},
{"path": "/home/user/Documents/Fundings/grf/customer-information-sheet-for-inward-payments-to-hong-kong.pdf", "url": "https://drive.google.com/uc?id=1s-H3an7HLBM9ku6d6Hcdj1qkSwKAHngU&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf15.pdf", "url": "https://drive.google.com/uc?id=1rRQXo9XHnCVTG8XqNAv0SJwPTW36MMbm&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf16.pdf", "url": "https://drive.google.com/uc?id=18ljRqkdyXEZ464E0dpKjaEa2NFexyw3I&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf17.pdf", "url": "https://drive.google.com/uc?id=1VrqOnyhpOkMpyIJ6YMrAhixpahjYonOd&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf18.pdf", "url": "https://drive.google.com/uc?id=182CLDUr372-jpAiY4YvSbGNXF9TsWxzA&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf19.pdf", "url": "https://drive.google.com/uc?id=1YkJtjlklKN0NmLiI2Hi4f_dKtTm5SPxT&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf20.pdf", "url": "https://drive.google.com/uc?id=1a7Uc7VCMlEX6fy-5oqE6i1YLitBe7gaf&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf21.pdf", "url": "https://drive.google.com/uc?id=1s8km4Wle4lc5PkbUQfivBFK0IJQgxMiB&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf22.pdf", "url": "https://drive.google.com/uc?id=1HTEb1uK7LNvbVyeXgO8WemCPPgiASKiy&export=download"},
{"path": "/home/user/Documents/Fundings/grf/grf23.pdf", "url": "https://drive.google.com/uc?id=1XM-jZlfu_i4waDZHb8Z6Vr5b3LgULTtP&export=download"}
]
}
}
],
"trajectory": "trajectories/7e287123-70ca-47b9-8521-47db09b69b14",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user/Desktop",
"/home/user/Desktop/GRF-p5y.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/Desktop/GRF-p5y.xlsx",
"/home/user/Desktop/GRF-p5y-Sheet1.csv"
],
"dest": [
"GRF-p5y.xlsx",
"GRF-p5y-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1fDM4Y-WuFCnfksPLgynj-WSmzbqn2TcV&export=download",
"https://drive.google.com/uc?id=1waThupubGOJop0FU0b0yhT6QnjNYkLGy&export=download"
],
"dest": [
"GRF-p5y-gt.xlsx",
"GRF-p5y-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1"
}
]
}
}
}