From 53ed5588e38ffa8a97e12a95f53ea505c5675080 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 1 Mar 2024 23:14:42 +0800 Subject: [PATCH] ver Mar1stv2 two new tasks --- .../00fa164e-2612-4439-992e-157d019a8436.json | 85 +++++++++++++++++ .../7e287123-70ca-47b9-8521-47db09b69b14.json | 2 +- .../b5062e3e-641c-4e3a-907b-ac864d2e7652.json | 94 +++++++++++++++++++ 3 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json create mode 100644 evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json diff --git a/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json b/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json new file mode 100644 index 0000000..3baf77f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/00fa164e-2612-4439-992e-157d019a8436.json @@ -0,0 +1,85 @@ +{ + "id": "00fa164e-2612-4439-992e-157d019a8436", + "snapshot": "libreoffice_writer", + "instruction": "I'm writing a report in Writer. A little data are recorded in \"~/Documents/awesome-desktop/expe-results.xlsx\". Help me to extract the results of LLM-based models and insert a table in the \"Main Results\" section of my report.", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": ["mkdir", "-p", "/home/user/Documents/awesome-desktop/"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx", + "url": "https://drive.google.com/uc?id=1oXro9_S7nZs_h_AZq5TJWEBofbrZM3oG&export=download" + }, + { + "path": "/home/user/Documents/awesome-desktop/expe-results.xlsx", + "url": "https://drive.google.com/uc?id=1DwOikfnAumkZfvVcjIncgKZMITY3ruku&export=download" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx" + } + } + ], + "trajectory": "trajectories/00fa164e-2612-4439-992e-157d019a8436", + "related_apps": [ + "libreoffice_writer", + "libreoffice_calc", + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "awe_desk_env.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 1.0 + } + } + ], + "func": "compare_docx_tables", + "result": { + "type": "vm_file", + "path": "/home/user/Documents/awesome-desktop/awe_desk_env.docx", + "dest": "awe_desk_env.docx" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.google.com/uc?id=1hMZKSF74qBUvBDAu_zF8vHm6MKu96VBN&export=download", + "dest": "awe_desk_env_gt.docx" + } + } +} diff --git a/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json b/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json index a398bd4..95d1eec 100644 --- a/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json +++ b/evaluation_examples/examples/multi_apps/7e287123-70ca-47b9-8521-47db09b69b14.json @@ -1,7 +1,7 @@ { "id": "7e287123-70ca-47b9-8521-47db09b69b14", "snapshot": "libreoffice_calc", - "instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for the past five years in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.", + "instruction": "I am an assistant professor of CS at HKU, I want to apply for the General Research Fund next year, I need to get some insights, so I need you to help me to organise the data. First please help me to organise the pass rate of the GRF applications of the CS departments of each school for 2019~2023 in percentage form with 2 decimal digits in a table, which I can use subsequently. Set the headers as \"Year\", \"#Applied\", \"#Supported\", and \"Success Rate\". The materials are saved under Documents/Fundings. And please save the result table as \"GRF-p5y.xlsx\" on my desktop.", "source": "authors", "config": [ { diff --git a/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json b/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json new file mode 100644 index 0000000..0b797a7 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/b5062e3e-641c-4e3a-907b-ac864d2e7652.json @@ -0,0 +1,94 @@ +{ + "id": "b5062e3e-641c-4e3a-907b-ac864d2e7652", + "snapshot": "libreoffice_calc", + "instruction": "I've got a mass of different categories of paper in PDF under folder \"~/Documents/Papers\". Please help me to extract the name, mail, and affiliation of the first author of all the papers and note them in an Excel table. The corresponding headers should be added in the table. Simply sort the authors by their full names ascendingly. The summary file should be saved as \"~/authors.xlsx\".", + "source": "authors", + "config": [ + { + "type": "command", + "parameters": { + "command": ["mkdir", "-p", "/home/user/Documents/Papers"] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "path": "/home/user/Documents/Papers/zhang_appagent.pdf", + "url": "https://arxiv.org/pdf/2312.13771.pdf" + }, + { + "path": "/home/user/Documents/Papers/niu_screenagent.pdf", + "url": "https://arxiv.org/pdf/2402.07945.pdf" + }, + { + "path": "/home/user/Documents/Papers/koh_visualwebarena.pdf", + "url": "https://arxiv.org/pdf/2401.13649.pdf" + }, + { + "path": "/home/user/Documents/Papers/deng_mind2web.pdf", + "url": "https://papers.nips.cc/paper_files/paper/2023/file/5950bf290a1570ea401bf98882128160-Paper-Datasets_and_Benchmarks.pdf" + } + ] + } + } + ], + "trajectory": "trajectories/b5062e3e-641c-4e3a-907b-ac864d2e7652", + "related_apps": [ + "libreoffice_calc", + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", + "--outdir", + "/home/user", + "/home/user/authors.xlsx" + ] + } + } + ], + "func": "compare_table", + "result": { + "type": "vm_file", + "path": [ + "/home/user/authors.xlsx", + "/home/user/authors-Sheet1.csv" + ], + "dest": [ + "authors.xlsx", + "authors-Sheet1.csv" + ], + "multi": true + }, + "expected": { + "type": "cloud_file", + "path": [ + "https://drive.google.com/uc?id=1fttbvfHuoQfsQUk3fVXkJsCu231jhnQj&export=download", + "https://drive.google.com/uc?id=1fq4hbk1g9R_SjknzwFAqvyF1ICyNYfok&export=download" + ], + "dest": [ + "authors-gt.xlsx", + "authors-gt-Sheet1.csv" + ], + "multi": true + }, + "options": { + "rules": [ + { + "type": "sheet_print", + "sheet_idx0": "RNSheet1", + "sheet_idx1": "ENSheet1", + "ignore_case": true + } + ] + } + } +}