diff --git a/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json b/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json new file mode 100644 index 0000000..3baf77f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json @@ -0,0 +1,85 @@ +{ + "id": "00fa164e-2612-4439-992e-157d019a8436", + "snapshot": "libreoffice_writer", + "instruction": "I'm writing a report in Writer. A little data are recorded in \"~/Documents/awesome-desktop/expe-results.xlsx\". Help me to extract the results of LLM-based models and insert a table in the \"Main Results\" section of my report.", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": ["mkdir", "-p", "/home/user/Documents/awesome-desktop/"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx", + "url": "https://drive.google.com/uc?id=1oXro9_S7nZs_h_AZq5TJWEBofbrZM3oG&export=download" + }, + { + "path": "/home/user/Documents/awesome-desktop/expe-results.xlsx", + "url": "https://drive.google.com/uc?id=1DwOikfnAumkZfvVcjIncgKZMITY3ruku&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx" + } + } + ], + "trajectory": "trajectories/00fa164e-2612-4439-992e-157d019a8436", + "related_apps": [ + "libreoffice_writer", + "libreoffice_calc", + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "awe_desk_env.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 1.0 + } + } + ], + "func": "compare_docx_tables", + "result": { + "type": "vm_file", + "path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx", + "dest": "awe_desk_env.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1hMZKSF74qBUvBDAu_zF8vHm6MKu96VBN&export=download", + "dest": "awe_desk_env_gt.docx" + } + } +} diff --git a/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json b/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json index a398bd4..95d1eec 100644 --- a/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json +++ b/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json @@ -1,7 +1,7 @@ { "id": "7e287123-70ca-47b9-8521-47db09b69b14", "snapshot": "libreoffice_calc", - "instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for the past five years in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.", + "instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for 2019~2023 in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.", "source": "authors", "config": [ { diff --git a/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json b/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json new file mode 100644 index 0000000..0b797a7 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json @@ -0,0 +1,94 @@ +{ + "id": "b5062e3e-641c-4e3a-907b-ac864d2e7652", + "snapshot": "libreoffice_calc", + "instruction": "I've got a mass of different categories of paper in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": ["mkdir", "-p", "/home/user/Documents/Papers"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "/home/user/Documents/Papers/zhang_appagent.pdf", + "url": "https://arxiv.org/pdf/2312.13771.pdf" + }, + { + "path": "/home/user/Documents/Papers/niu_screenagent.pdf", + "url": "https://arxiv.org/pdf/2402.07945.pdf" + }, + { + "path": "/home/user/Documents/Papers/koh_visualwebarena.pdf", + "url": "https://arxiv.org/pdf/2401.13649.pdf" + }, + { + "path": "/home/user/Documents/Papers/deng_mind2web.pdf", + "url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf" + } + ] + } + } + ], + "trajectory": "trajectories/b5062e3e-641c-4e3a-907b-ac864d2e7652", + "related_apps": [ + "libreoffice_calc", + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", + "--outdir", + "/home/user", + "/home/user/authors.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/authors.xlsx", + "/home/user/authors-Sheet1.csv" + ], + "dest": [ + "authors.xlsx", + "authors-Sheet1.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.google.com/uc?id=1fttbvfHuoQfsQUk3fVXkJsCu231jhnQj&export=download", + "https://drive.google.com/uc?id=1fq4hbk1g9R_SjknzwFAqvyF1ICyNYfok&export=download" + ], + "dest": [ + "authors-gt.xlsx", + "authors-gt-Sheet1.csv" + ], + "multi": true + }, + "options": { + "rules": [ + { + "type": "sheet_print", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "ignore_case": true + } + ] + } + } +}