lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-03-01 23:15:48 +08:00
parent 273bd16c3a 53ed5588e3
commit 6926dfbc5f
3 changed files with 180 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json Normal file
View File

@@ -0,0 +1,85 @@
{
"id": "00fa164e-2612-4439-992e-157d019a8436",
"snapshot": "libreoffice_writer",
"instruction": "I'm writing a report in Writer. A little data are recorded in \"~/Documents/awesome-desktop/expe-results.xlsx\". Help me to extract the results of LLM-based models and insert a table in the \"Main Results\" section of my report.",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/awesome-desktop/"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx",
"url": "https://drive.google.com/uc?id=1oXro9_S7nZs_h_AZq5TJWEBofbrZM3oG&export=download"
},
{
"path": "/home/user/Documents/awesome-desktop/expe-results.xlsx",
"url": "https://drive.google.com/uc?id=1DwOikfnAumkZfvVcjIncgKZMITY3ruku&export=download"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx"
}
}
],
"trajectory": "trajectories/00fa164e-2612-4439-992e-157d019a8436",
"related_apps": [
"libreoffice_writer",
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "awe_desk_env.docx - LibreOffice Writer",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 1.0
}
}
],
"func": "compare_docx_tables",
"result": {
"type": "vm_file",
"path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx",
"dest": "awe_desk_env.docx"
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?id=1hMZKSF74qBUvBDAu_zF8vHm6MKu96VBN&export=download",
"dest": "awe_desk_env_gt.docx"
}
}
}

2
evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json
View File

@@ -1,7 +1,7 @@
{
"id": "7e287123-70ca-47b9-8521-47db09b69b14",
"snapshot": "libreoffice_calc",
"instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for the past five years in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.",
"instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for 2019~2023 in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.",
"source": "authors",
"config": [
{

94
evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json Normal file
View File

@@ -0,0 +1,94 @@
{
"id": "b5062e3e-641c-4e3a-907b-ac864d2e7652",
"snapshot": "libreoffice_calc",
"instruction": "I've got a mass of different categories of paper in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".",
"source": "authors",
"config": [
{
"type": "command",
"parameters": {
"command": ["mkdir", "-p", "/home/user/Documents/Papers"]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"path": "/home/user/Documents/Papers/zhang_appagent.pdf",
"url": "https://arxiv.org/pdf/2312.13771.pdf"
},
{
"path": "/home/user/Documents/Papers/niu_screenagent.pdf",
"url": "https://arxiv.org/pdf/2402.07945.pdf"
},
{
"path": "/home/user/Documents/Papers/koh_visualwebarena.pdf",
"url": "https://arxiv.org/pdf/2401.13649.pdf"
},
{
"path": "/home/user/Documents/Papers/deng_mind2web.pdf",
"url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf"
}
]
}
}
],
"trajectory": "trajectories/b5062e3e-641c-4e3a-907b-ac864d2e7652",
"related_apps": [
"libreoffice_calc",
"os"
],
"evaluator": {
"postconfig": [
{
"type": "execute",
"parameters": {
"command": [
"libreoffice",
"--convert-to",
"csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1",
"--outdir",
"/home/user",
"/home/user/authors.xlsx"
]
}
}
],
"func": "compare_table",
"result": {
"type": "vm_file",
"path": [
"/home/user/authors.xlsx",
"/home/user/authors-Sheet1.csv"
],
"dest": [
"authors.xlsx",
"authors-Sheet1.csv"
],
"multi": true
},
"expected": {
"type": "cloud_file",
"path": [
"https://drive.google.com/uc?id=1fttbvfHuoQfsQUk3fVXkJsCu231jhnQj&export=download",
"https://drive.google.com/uc?id=1fq4hbk1g9R_SjknzwFAqvyF1ICyNYfok&export=download"
],
"dest": [
"authors-gt.xlsx",
"authors-gt-Sheet1.csv"
],
"multi": true
},
"options": {
"rules": [
{
"type": "sheet_print",
"sheet_idx0": "RNSheet1",
"sheet_idx1": "ENSheet1",
"ignore_case": true
}
]
}
}
}